2019-11-07 17:49:26 +01:00
"""
2019-11-08 11:05:34 +01:00
Handle the email * forward * and * reply * . phase . There are 3 actors :
2020-03-26 11:19:20 +01:00
- contact : who sends emails to alias @sl.co address
2019-11-07 17:49:26 +01:00
- SL email handler ( this script )
2020-03-26 11:19:20 +01:00
- user personal email : to be protected . Should never leak to contact .
2019-11-07 17:49:26 +01:00
This script makes sure that in the forward phase , the email that is forwarded to user personal email has the following
envelope and header fields :
Envelope :
2020-03-26 11:19:20 +01:00
mail from : @contact
2019-11-08 11:05:34 +01:00
rcpt to : @personal_email
2019-11-07 17:49:26 +01:00
Header :
2020-03-26 11:19:20 +01:00
From : @contact
2019-11-08 11:05:34 +01:00
To : alias @sl.co # so user knows this email is sent to alias
Reply - to : special @sl.co # magic HERE
2019-11-07 17:49:26 +01:00
And in the reply phase :
Envelope :
2020-03-26 11:19:20 +01:00
mail from : @contact
rcpt to : @contact
2019-11-07 17:49:26 +01:00
Header :
2020-03-26 11:19:20 +01:00
From : alias @sl.co # so for contact the email comes from alias. magic HERE
To : @contact
2019-11-07 17:49:26 +01:00
The special @sl.co allows to hide user personal email when user clicks " Reply " to the forwarded email .
It should contain the following info :
- alias
2020-03-26 11:19:20 +01:00
- @contact
2019-11-07 17:49:26 +01:00
"""
2020-09-02 17:36:11 +02:00
import argparse
2020-04-02 18:10:08 +02:00
import email
2019-11-07 17:49:26 +01:00
import time
2020-03-15 22:29:53 +01:00
import uuid
2020-03-08 23:07:23 +01:00
from email import encoders
2020-11-02 19:09:57 +01:00
from email . encoders import encode_noop
2020-01-04 10:25:19 +01:00
from email . message import Message
2020-03-08 23:07:23 +01:00
from email . mime . application import MIMEApplication
from email . mime . multipart import MIMEMultipart
2020-11-21 19:15:02 +01:00
from email . utils import formataddr , make_msgid , formatdate , getaddresses
2020-03-14 16:34:23 +01:00
from io import BytesIO
2021-12-23 19:34:17 +01:00
from smtplib import SMTPRecipientsRefused , SMTPServerDisconnected
2020-09-28 17:41:16 +02:00
from typing import List , Tuple , Optional
2019-11-07 17:49:26 +01:00
2021-07-23 15:48:43 +02:00
import newrelic . agent
2020-09-30 11:05:21 +02:00
from aiosmtpd . controller import Controller
2020-06-07 11:41:35 +02:00
from aiosmtpd . smtp import Envelope
2021-09-10 16:51:36 +02:00
from email_validator import validate_email , EmailNotValidError
from flanker . addresslib import address
from flanker . addresslib . address import EmailAddress
2020-09-03 19:42:52 +02:00
from sqlalchemy . exc import IntegrityError
2021-12-31 12:14:22 +01:00
from sqlalchemy . orm . exc import ObjectDeletedError
2020-06-07 11:41:35 +02:00
2020-03-14 16:34:23 +01:00
from app import pgp_utils , s3
2020-04-04 15:24:27 +02:00
from app . alias_utils import try_auto_create
2020-02-15 11:04:22 +01:00
from app . config import (
EMAIL_DOMAIN ,
URL ,
2020-03-28 23:19:25 +01:00
UNSUBSCRIBER ,
2020-04-14 12:45:47 +02:00
LOAD_PGP_EMAIL_HANDLER ,
2020-05-07 13:28:04 +02:00
ENFORCE_SPF ,
2020-05-09 20:45:04 +02:00
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX ,
ALERT_BOUNCE_EMAIL ,
ALERT_SPAM_EMAIL ,
2020-08-15 16:38:16 +02:00
SPAMASSASSIN_HOST ,
MAX_SPAM_SCORE ,
2020-08-15 16:53:57 +02:00
MAX_REPLY_PHASE_SPAM_SCORE ,
2020-08-25 12:51:05 +02:00
ALERT_SEND_EMAIL_CYCLE ,
2020-08-30 19:06:50 +02:00
ALERT_MAILBOX_IS_ALIAS ,
2020-11-02 19:09:57 +01:00
PGP_SENDER_PRIVATE_KEY ,
2020-11-04 12:32:15 +01:00
ALERT_BOUNCE_EMAIL_REPLY_PHASE ,
2020-11-14 15:55:53 +01:00
NOREPLY ,
2021-01-11 14:55:55 +01:00
BOUNCE_EMAIL ,
BOUNCE_PREFIX ,
BOUNCE_SUFFIX ,
2021-01-26 09:59:08 +01:00
TRANSACTIONAL_BOUNCE_PREFIX ,
TRANSACTIONAL_BOUNCE_SUFFIX ,
2021-03-26 10:00:48 +01:00
ENABLE_SPAM_ASSASSIN ,
2021-05-25 17:58:45 +02:00
BOUNCE_PREFIX_FOR_REPLY_PHASE ,
2021-09-06 19:44:18 +02:00
POSTMASTER ,
ALERT_HOTMAIL_COMPLAINT ,
2021-09-13 19:49:40 +02:00
ALERT_YAHOO_COMPLAINT ,
2021-11-21 11:31:28 +01:00
ALERT_HOTMAIL_COMPLAINT_TRANSACTIONAL ,
ALERT_HOTMAIL_COMPLAINT_REPLY_PHASE ,
2021-11-22 18:17:07 +01:00
OLD_UNSUBSCRIBER ,
2022-01-08 00:42:03 +01:00
ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS ,
2022-02-16 18:38:31 +01:00
ALERT_TO_NOREPLY ,
2022-03-21 17:43:26 +01:00
DMARC_CHECK_ENABLED ,
2022-03-25 17:25:35 +01:00
ALERT_QUARANTINE_DMARC ,
2020-02-15 11:04:22 +01:00
)
2021-10-12 14:36:47 +02:00
from app . db import Session
2021-10-11 12:00:37 +02:00
from app . email import status , headers
2021-07-23 15:48:43 +02:00
from app . email . rate_limit import rate_limited
2021-03-17 10:07:13 +01:00
from app . email . spam import get_spam_score
2019-12-17 17:48:06 +01:00
from app . email_utils import (
send_email ,
2021-09-25 18:47:15 +02:00
add_dkim_signature ,
2020-01-07 19:50:36 +01:00
add_or_replace_header ,
delete_header ,
2020-02-11 16:46:53 +01:00
render ,
2020-03-14 16:34:23 +01:00
get_orig_message_from_bounce ,
2020-03-14 22:24:02 +01:00
delete_all_headers_except ,
2020-03-30 22:05:31 +02:00
get_spam_info ,
get_orig_message_from_spamassassin_report ,
2020-05-09 20:45:04 +02:00
send_email_with_rate_control ,
2020-06-09 17:16:32 +02:00
get_email_domain_part ,
2020-08-21 12:03:23 +02:00
copy ,
to_bytes ,
2020-08-27 10:43:48 +02:00
send_email_at_most_times ,
2020-10-15 16:21:31 +02:00
is_valid_alias_address_domain ,
2021-09-25 18:47:15 +02:00
should_add_dkim_signature ,
2020-11-07 13:00:45 +01:00
add_header ,
2020-11-09 21:16:50 +01:00
get_header_unicode ,
2020-11-16 19:15:09 +01:00
generate_reply_email ,
2021-10-19 12:14:16 +02:00
is_reverse_alias ,
2020-11-22 13:07:09 +01:00
normalize_reply_email ,
2020-11-25 15:20:42 +01:00
is_valid_email ,
2020-11-30 15:15:44 +01:00
replace ,
2020-12-16 18:48:10 +01:00
should_disable ,
2021-01-26 09:46:47 +01:00
parse_id_from_bounce ,
2021-03-17 10:09:02 +01:00
spf_pass ,
sl_sendmail ,
2021-03-17 10:59:13 +01:00
sanitize_header ,
2021-06-04 17:15:59 +02:00
get_queue_id ,
2021-08-02 11:33:58 +02:00
should_ignore_bounce ,
2021-09-13 19:50:15 +02:00
get_orig_message_from_hotmail_complaint ,
2021-09-10 17:31:29 +02:00
parse_full_address ,
2021-09-13 19:49:40 +02:00
get_orig_message_from_yahoo_complaint ,
2021-10-14 15:46:52 +02:00
get_mailbox_bounce_info ,
2021-11-02 14:32:16 +01:00
save_email_for_debugging ,
2022-03-21 12:03:11 +01:00
get_dmarc_status ,
2020-01-07 19:50:36 +01:00
)
2022-01-07 14:57:47 +01:00
from app . errors import (
NonReverseAliasInReplyPhase ,
VERPTransactional ,
VERPForward ,
VERPReply ,
2022-01-08 00:16:16 +01:00
CannotCreateContactForReverseAlias ,
2022-01-07 14:57:47 +01:00
)
2021-03-15 19:41:42 +01:00
from app . log import LOG , set_message_id
2020-01-30 08:43:31 +01:00
from app . models import (
2020-03-17 11:51:40 +01:00
Alias ,
2020-03-17 10:56:59 +01:00
Contact ,
2022-02-21 12:52:21 +01:00
BlockBehaviourEnum ,
2020-03-17 11:10:50 +01:00
EmailLog ,
2020-01-30 08:43:31 +01:00
User ,
2020-03-14 16:34:23 +01:00
RefusedEmail ,
2020-05-07 13:28:04 +02:00
Mailbox ,
2021-01-26 09:56:13 +01:00
Bounce ,
TransactionalEmail ,
2021-06-22 17:52:24 +02:00
IgnoredEmail ,
2021-10-18 17:34:11 +02:00
MessageIDMatching ,
2022-01-16 11:47:50 +01:00
DeletedAlias ,
DomainDeletedAlias ,
2022-01-24 16:10:36 +01:00
Notification ,
2022-03-21 12:03:11 +01:00
DmarcCheckResult ,
2020-01-30 08:43:31 +01:00
)
2020-11-02 19:09:57 +01:00
from app . pgp_utils import PGPException , sign_data_with_pgpy , sign_data
2021-03-17 10:59:13 +01:00
from app . utils import sanitize_email
2020-04-14 12:45:47 +02:00
from init_app import load_pgp_public_keys
2021-10-12 15:03:16 +02:00
from server import create_light_app
2019-11-07 17:49:26 +01:00
2020-11-02 19:09:57 +01:00
2022-01-07 13:02:16 +01:00
def get_or_create_contact (
from_header : str , mail_from : str , alias : Alias , msg : Message
) - > Contact :
2020-02-19 16:49:40 +01:00
"""
2020-04-05 15:24:09 +02:00
contact_from_header is the RFC 2047 format FROM header
2020-02-19 16:49:40 +01:00
"""
2021-09-10 17:31:29 +02:00
try :
contact_name , contact_email = parse_full_address ( from_header )
except ValueError :
contact_name , contact_email = " " , " "
2020-11-25 17:50:25 +01:00
if not is_valid_email ( contact_email ) :
2020-05-15 15:46:37 +02:00
# From header is wrongly formatted, try with mail_from
2020-11-14 15:55:53 +01:00
if mail_from and mail_from != " <> " :
2021-03-16 09:17:23 +01:00
LOG . w (
2021-09-10 17:06:38 +02:00
" Cannot parse email from from_header %s , use mail_from %s " ,
2020-12-05 18:15:53 +01:00
from_header ,
2020-11-25 15:20:00 +01:00
mail_from ,
)
2021-09-10 17:06:38 +02:00
contact_email = mail_from
2020-05-13 22:35:27 +02:00
2020-11-25 17:50:25 +01:00
if not is_valid_email ( contact_email ) :
2021-03-16 09:17:23 +01:00
LOG . w (
2020-11-25 15:20:42 +01:00
" invalid contact email %s . Parse from %s %s " ,
contact_email ,
2020-12-05 18:15:53 +01:00
from_header ,
2020-11-25 15:20:42 +01:00
mail_from ,
)
# either reuse a contact with empty email or create a new contact with empty email
contact_email = " "
2021-11-18 16:44:04 +01:00
contact_email = sanitize_email ( contact_email , not_lower = True )
2021-01-11 12:27:02 +01:00
2021-09-27 10:21:49 +02:00
if contact_name and " \x00 " in contact_name :
LOG . w ( " issue with contact name %s " , contact_name )
contact_name = " "
2020-04-04 20:06:35 +02:00
contact = Contact . get_by ( alias_id = alias . id , website_email = contact_email )
2020-03-17 10:56:59 +01:00
if contact :
2020-04-05 15:24:09 +02:00
if contact . name != contact_name :
LOG . d (
2020-08-27 10:20:48 +02:00
" Update contact %s name %s to %s " ,
contact ,
contact . name ,
contact_name ,
2020-04-05 15:24:09 +02:00
)
contact . name = contact_name
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-10-27 10:40:44 +01:00
2020-11-25 15:20:00 +01:00
# contact created in the past does not have mail_from and from_header field
if not contact . mail_from and mail_from :
2020-10-27 10:40:44 +01:00
LOG . d (
2020-11-25 15:20:00 +01:00
" Set contact mail_from %s : %s to %s " ,
2020-10-27 10:40:44 +01:00
contact ,
contact . mail_from ,
mail_from ,
)
contact . mail_from = mail_from
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-02-19 16:17:13 +01:00
else :
2020-02-19 15:50:38 +01:00
2020-09-03 19:42:52 +02:00
try :
contact = Contact . create (
user_id = alias . user_id ,
alias_id = alias . id ,
website_email = contact_email ,
name = contact_name ,
2020-09-14 18:22:26 +02:00
mail_from = mail_from ,
2020-12-06 19:37:20 +01:00
reply_email = generate_reply_email ( contact_email , alias . user )
2020-11-25 17:50:25 +01:00
if is_valid_email ( contact_email )
2020-11-18 11:48:09 +01:00
else NOREPLY ,
2022-01-07 10:22:02 +01:00
automatic_created = True ,
2020-09-03 19:42:52 +02:00
)
2020-11-18 11:48:09 +01:00
if not contact_email :
2020-11-14 15:55:53 +01:00
LOG . d ( " Create a contact with invalid email for %s " , alias )
contact . invalid_email = True
2022-01-07 10:04:12 +01:00
LOG . d (
" create contact %s for %s , reverse alias: %s " ,
contact_email ,
alias ,
contact . reply_email ,
)
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-09-03 19:42:52 +02:00
except IntegrityError :
2021-03-16 09:17:23 +01:00
LOG . w ( " Contact %s %s already exist " , alias , contact_email )
2021-10-12 14:36:47 +02:00
Session . rollback ( )
2020-09-03 19:42:52 +02:00
contact = Contact . get_by ( alias_id = alias . id , website_email = contact_email )
2020-02-19 15:50:38 +01:00
2020-03-17 10:56:59 +01:00
return contact
2020-02-19 16:49:40 +01:00
2021-07-28 09:12:52 +02:00
def get_or_create_reply_to_contact (
2022-01-07 13:02:16 +01:00
reply_to_header : str , alias : Alias , msg : Message
2021-07-28 09:12:52 +02:00
) - > Optional [ Contact ] :
"""
Get or create the contact for the Reply - To header
"""
2021-09-10 17:31:29 +02:00
try :
contact_name , contact_address = parse_full_address ( reply_to_header )
except ValueError :
return
2021-07-28 09:12:52 +02:00
2021-09-10 17:31:29 +02:00
if not is_valid_email ( contact_address ) :
2021-07-28 09:12:52 +02:00
LOG . w (
" invalid reply-to address %s . Parse from %s " ,
2021-09-10 17:31:29 +02:00
contact_address ,
2021-07-28 09:12:52 +02:00
reply_to_header ,
)
return None
2021-09-10 17:31:29 +02:00
contact = Contact . get_by ( alias_id = alias . id , website_email = contact_address )
2021-07-28 09:12:52 +02:00
if contact :
return contact
else :
LOG . d (
" create contact %s for alias %s via reply-to header " ,
2021-09-10 17:31:29 +02:00
contact_address ,
2021-07-28 09:12:52 +02:00
alias ,
reply_to_header ,
)
try :
contact = Contact . create (
user_id = alias . user_id ,
alias_id = alias . id ,
2021-09-10 17:31:29 +02:00
website_email = contact_address ,
name = contact_name ,
reply_email = generate_reply_email ( contact_address , alias . user ) ,
2022-01-07 10:22:02 +01:00
automatic_created = True ,
2021-07-28 09:12:52 +02:00
)
2021-10-12 14:36:47 +02:00
Session . commit ( )
2021-07-28 09:12:52 +02:00
except IntegrityError :
2021-09-10 17:31:29 +02:00
LOG . w ( " Contact %s %s already exist " , alias , contact_address )
2021-10-12 14:36:47 +02:00
Session . rollback ( )
2021-09-10 17:31:29 +02:00
contact = Contact . get_by ( alias_id = alias . id , website_email = contact_address )
2021-07-28 09:12:52 +02:00
return contact
2020-03-28 19:16:55 +01:00
def replace_header_when_forward ( msg : Message , alias : Alias , header : str ) :
"""
Replace CC or To header by Reply emails in forward phase
"""
new_addrs : [ str ] = [ ]
2020-11-26 09:49:03 +01:00
headers = msg . get_all ( header , [ ] )
2021-01-15 11:30:43 +01:00
# headers can be an array of Header, convert it to string here
2021-09-10 16:51:36 +02:00
headers = [ get_header_unicode ( h ) for h in headers ]
2021-01-11 12:27:02 +01:00
2021-09-10 16:51:36 +02:00
full_addresses : [ EmailAddress ] = [ ]
for h in headers :
full_addresses + = address . parse_list ( h )
for full_address in full_addresses :
2021-11-18 16:44:04 +01:00
contact_email = sanitize_email ( full_address . address , not_lower = True )
2020-03-28 19:16:55 +01:00
# no transformation when alias is already in the header
2021-11-18 16:44:04 +01:00
if contact_email . lower ( ) == alias . email :
2021-09-10 16:51:36 +02:00
new_addrs . append ( full_address . full_spec ( ) )
2020-03-28 19:16:55 +01:00
continue
2021-09-10 16:51:36 +02:00
try :
# NOT allow unicode for contact address
2021-09-10 17:06:38 +02:00
validate_email (
contact_email , check_deliverability = False , allow_smtputf8 = False
)
2021-09-10 16:51:36 +02:00
except EmailNotValidError :
2021-03-16 09:17:23 +01:00
LOG . w ( " invalid contact email %s . %s . Skip " , contact_email , headers )
2020-11-25 15:21:01 +01:00
continue
2020-04-05 12:59:36 +02:00
contact = Contact . get_by ( alias_id = alias . id , website_email = contact_email )
2020-03-28 19:16:55 +01:00
if contact :
2020-04-05 15:39:48 +02:00
# update the contact name if needed
2021-09-10 16:51:36 +02:00
if contact . name != full_address . display_name :
2020-04-05 14:50:12 +02:00
LOG . d (
" Update contact %s name %s to %s " ,
contact ,
contact . name ,
2021-09-10 16:51:36 +02:00
full_address . display_name ,
2020-04-05 14:50:12 +02:00
)
2021-09-10 16:51:36 +02:00
contact . name = full_address . display_name
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-03-28 19:16:55 +01:00
else :
2021-09-08 11:29:55 +02:00
LOG . d (
2020-03-28 19:16:55 +01:00
" create contact for alias %s and email %s , header %s " ,
alias ,
2020-04-05 12:59:36 +02:00
contact_email ,
2020-03-28 19:16:55 +01:00
header ,
)
2020-09-09 17:00:07 +02:00
try :
contact = Contact . create (
user_id = alias . user_id ,
alias_id = alias . id ,
website_email = contact_email ,
2021-09-10 16:51:36 +02:00
name = full_address . display_name ,
2020-12-06 19:37:20 +01:00
reply_email = generate_reply_email ( contact_email , alias . user ) ,
2020-09-09 17:00:07 +02:00
is_cc = header . lower ( ) == " cc " ,
2022-01-07 10:22:02 +01:00
automatic_created = True ,
2020-09-09 17:00:07 +02:00
)
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-09-09 17:00:07 +02:00
except IntegrityError :
2021-03-16 09:17:23 +01:00
LOG . w ( " Contact %s %s already exist " , alias , contact_email )
2021-10-12 14:36:47 +02:00
Session . rollback ( )
2020-09-09 17:00:07 +02:00
contact = Contact . get_by ( alias_id = alias . id , website_email = contact_email )
2020-03-28 19:16:55 +01:00
2020-04-05 15:24:09 +02:00
new_addrs . append ( contact . new_addr ( ) )
2020-03-28 19:16:55 +01:00
2020-08-25 12:48:28 +02:00
if new_addrs :
2020-03-28 19:16:55 +01:00
new_header = " , " . join ( new_addrs )
LOG . d ( " Replace %s header, old: %s , new: %s " , header , msg [ header ] , new_header )
add_or_replace_header ( msg , header , new_header )
else :
2020-08-25 12:48:28 +02:00
LOG . d ( " Delete %s header, old value %s " , header , msg [ header ] )
delete_header ( msg , header )
2020-03-28 19:16:55 +01:00
def replace_header_when_reply ( msg : Message , alias : Alias , header : str ) :
"""
Replace CC or To Reply emails by original emails
"""
new_addrs : [ str ] = [ ]
2021-01-15 11:30:43 +01:00
headers = msg . get_all ( header , [ ] )
# headers can be an array of Header, convert it to string here
headers = [ str ( h ) for h in headers ]
2020-03-28 19:16:55 +01:00
2022-01-11 13:11:28 +01:00
# headers can contain \r or \n
headers = [ h . replace ( " \r " , " " ) for h in headers ]
headers = [ h . replace ( " \n " , " " ) for h in headers ]
2021-01-15 11:30:43 +01:00
for _ , reply_email in getaddresses ( headers ) :
2020-03-28 19:16:55 +01:00
# no transformation when alias is already in the header
2022-01-07 10:34:08 +01:00
# can happen when user clicks "Reply All"
2020-04-05 15:27:35 +02:00
if reply_email == alias . email :
2020-03-28 19:16:55 +01:00
continue
2020-04-05 15:27:35 +02:00
contact = Contact . get_by ( reply_email = reply_email )
2020-03-28 19:16:55 +01:00
if not contact :
2021-03-16 09:17:23 +01:00
LOG . w (
2022-01-11 12:29:42 +01:00
" email %s contained in %s header in reply phase must be reply emails. headers: %s " ,
2022-01-07 10:34:08 +01:00
reply_email ,
header ,
headers ,
2020-03-30 21:45:18 +02:00
)
2022-01-07 17:53:06 +01:00
raise NonReverseAliasInReplyPhase ( reply_email )
2020-03-28 19:16:55 +01:00
# still keep this email in header
2022-01-07 10:34:08 +01:00
# new_addrs.append(reply_email)
2020-04-05 15:27:35 +02:00
else :
new_addrs . append ( formataddr ( ( contact . name , contact . website_email ) ) )
2020-03-28 19:16:55 +01:00
2020-08-25 12:48:28 +02:00
if new_addrs :
new_header = " , " . join ( new_addrs )
LOG . d ( " Replace %s header, old: %s , new: %s " , header , msg [ header ] , new_header )
add_or_replace_header ( msg , header , new_header )
else :
LOG . d ( " delete the %s header. Old value %s " , header , msg [ header ] )
delete_header ( msg , header )
2020-03-28 19:16:55 +01:00
2020-11-02 19:09:57 +01:00
def prepare_pgp_message (
orig_msg : Message , pgp_fingerprint : str , public_key : str , can_sign : bool = False
2020-11-07 13:00:45 +01:00
) - > Message :
2020-03-08 23:07:23 +01:00
msg = MIMEMultipart ( " encrypted " , protocol = " application/pgp-encrypted " )
2020-11-01 18:06:05 +01:00
# clone orig message to avoid modifying it
clone_msg = copy ( orig_msg )
2020-04-14 20:49:48 +02:00
# copy all headers from original message except all standard MIME headers
2020-11-01 18:06:05 +01:00
for i in reversed ( range ( len ( clone_msg . _headers ) ) ) :
header_name = clone_msg . _headers [ i ] [ 0 ] . lower ( )
2021-10-11 12:19:21 +02:00
if header_name . lower ( ) not in headers . MIME_HEADERS :
2020-11-01 18:06:05 +01:00
msg [ header_name ] = clone_msg . _headers [ i ] [ 1 ]
2020-03-08 23:07:23 +01:00
2020-11-02 19:09:57 +01:00
# Delete unnecessary headers in clone_msg except _MIME_HEADERS to save space
2020-03-14 22:24:02 +01:00
delete_all_headers_except (
2020-11-01 18:06:05 +01:00
clone_msg ,
2021-10-11 12:19:21 +02:00
headers . MIME_HEADERS ,
2020-03-14 22:24:02 +01:00
)
2021-10-11 12:10:18 +02:00
if clone_msg [ headers . CONTENT_TYPE ] is None :
2020-11-01 18:06:28 +01:00
LOG . d ( " Content-Type missing " )
2021-10-11 12:10:18 +02:00
clone_msg [ headers . CONTENT_TYPE ] = " text/plain "
2020-11-01 18:06:28 +01:00
2021-10-11 12:10:18 +02:00
if clone_msg [ headers . MIME_VERSION ] is None :
2020-11-01 18:06:28 +01:00
LOG . d ( " Mime-Version missing " )
2021-10-11 12:10:18 +02:00
clone_msg [ headers . MIME_VERSION ] = " 1.0 "
2020-11-01 18:06:28 +01:00
2020-03-08 23:07:23 +01:00
first = MIMEApplication (
_subtype = " pgp-encrypted " , _encoder = encoders . encode_7or8bit , _data = " "
)
first . set_payload ( " Version: 1 " )
msg . attach ( first )
2020-11-02 19:09:57 +01:00
if can_sign and PGP_SENDER_PRIVATE_KEY :
LOG . d ( " Sign msg " )
clone_msg = sign_msg ( clone_msg )
# use pgpy as fallback
2020-09-08 11:10:22 +02:00
second = MIMEApplication (
" octet-stream " , _encoder = encoders . encode_7or8bit , name = " encrypted.asc "
)
second . add_header ( " Content-Disposition " , ' inline; filename= " encrypted.asc " ' )
2020-10-28 11:50:14 +01:00
2020-11-02 19:09:57 +01:00
# encrypt
2020-10-28 17:07:53 +01:00
# use pgpy as fallback
2020-11-09 17:02:10 +01:00
msg_bytes = to_bytes ( clone_msg )
2020-10-28 17:07:53 +01:00
try :
2020-11-01 18:06:05 +01:00
encrypted_data = pgp_utils . encrypt_file ( BytesIO ( msg_bytes ) , pgp_fingerprint )
2020-10-28 17:07:53 +01:00
second . set_payload ( encrypted_data )
except PGPException :
2021-03-16 09:17:23 +01:00
LOG . w ( " Cannot encrypt using python-gnupg, use pgpy " )
2020-11-01 18:06:05 +01:00
encrypted = pgp_utils . encrypt_file_with_pgpy ( msg_bytes , public_key )
2020-10-28 17:07:53 +01:00
second . set_payload ( str ( encrypted ) )
2020-10-28 11:50:14 +01:00
2020-03-08 23:07:23 +01:00
msg . attach ( second )
return msg
2020-11-02 19:09:57 +01:00
def sign_msg ( msg : Message ) - > Message :
container = MIMEMultipart (
" signed " , protocol = " application/pgp-signature " , micalg = " pgp-sha256 "
)
container . attach ( msg )
signature = MIMEApplication (
_subtype = " pgp-signature " , name = " signature.asc " , _data = " " , _encoder = encode_noop
)
signature . add_header ( " Content-Disposition " , ' attachment; filename= " signature.asc " ' )
try :
2020-11-09 17:02:10 +01:00
signature . set_payload ( sign_data ( to_bytes ( msg ) . replace ( b " \n " , b " \r \n " ) ) )
2020-11-02 19:09:57 +01:00
except Exception :
2021-09-08 11:29:55 +02:00
LOG . e ( " Cannot sign, try using pgpy " )
2020-11-02 19:09:57 +01:00
signature . set_payload (
2020-11-09 17:02:10 +01:00
sign_data_with_pgpy ( to_bytes ( msg ) . replace ( b " \n " , b " \r \n " ) )
2020-11-02 19:09:57 +01:00
)
container . attach ( signature )
return container
2022-01-08 00:09:45 +01:00
def handle_email_sent_to_ourself ( alias , from_addr : str , msg : Message , user ) :
2020-08-25 12:51:05 +02:00
# store the refused email
random_name = str ( uuid . uuid4 ( ) )
full_report_path = f " refused-emails/cycle- { random_name } .eml "
2020-11-09 17:02:10 +01:00
s3 . upload_email_from_bytesio ( full_report_path , BytesIO ( to_bytes ( msg ) ) , random_name )
2020-08-25 12:51:05 +02:00
refused_email = RefusedEmail . create (
path = None , full_report_path = full_report_path , user_id = alias . user_id
)
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-08-25 12:51:05 +02:00
LOG . d ( " Create refused email %s " , refused_email )
# link available for 6 days as it gets deleted in 7 days
refused_email_url = refused_email . get_url ( expires_in = 518400 )
2020-08-27 10:43:48 +02:00
send_email_at_most_times (
2020-08-25 12:51:05 +02:00
user ,
ALERT_SEND_EMAIL_CYCLE ,
2022-01-08 00:09:45 +01:00
from_addr ,
f " Email sent to { alias . email } from its own mailbox { from_addr } " ,
2020-08-25 12:51:05 +02:00
render (
2021-11-02 11:46:41 +01:00
" transactional/cycle-email.txt.jinja2 " ,
2020-08-25 12:51:05 +02:00
alias = alias ,
2022-01-08 00:09:45 +01:00
from_addr = from_addr ,
2020-08-25 12:51:05 +02:00
refused_email_url = refused_email_url ,
) ,
render (
" transactional/cycle-email.html " ,
alias = alias ,
2022-01-08 00:09:45 +01:00
from_addr = from_addr ,
2020-08-25 12:51:05 +02:00
refused_email_url = refused_email_url ,
) ,
)
2022-03-21 19:05:15 +01:00
def apply_dmarc_policy (
2022-03-25 17:25:35 +01:00
alias : Alias , contact : Contact , envelope : Envelope , msg : Message , from_header
2022-03-21 19:05:15 +01:00
) - > Optional [ str ] :
2022-03-21 12:03:11 +01:00
dmarc_result = get_dmarc_status ( msg )
2022-03-23 08:34:25 +01:00
if dmarc_result :
2022-03-25 16:49:49 +01:00
newrelic . agent . record_custom_event ( " DmarcCheck " , { " result " : dmarc_result . name } )
2022-03-25 16:19:11 +01:00
else :
2022-03-25 18:00:49 +01:00
newrelic . agent . record_custom_event ( " DmarcCheck " , { " result " : " unknown " } )
2022-03-22 18:54:45 +01:00
2022-03-23 08:34:25 +01:00
if not DMARC_CHECK_ENABLED or not dmarc_result :
2022-03-21 12:03:11 +01:00
return None
2022-03-22 18:54:45 +01:00
2022-03-21 12:03:11 +01:00
if dmarc_result in (
DmarcCheckResult . quarantine ,
DmarcCheckResult . reject ,
2022-03-21 17:38:41 +01:00
DmarcCheckResult . soft_fail ,
2022-03-18 15:44:07 +01:00
) :
2022-03-25 16:20:30 +01:00
LOG . w (
f " put email from { contact } to { alias } to quarantine. { dmarc_result } , "
f " mail_from: { envelope . mail_from } , from_header: { msg [ headers . FROM ] } "
)
2022-03-25 17:25:35 +01:00
email_log = quarantine_dmarc_failed_email ( alias , contact , envelope , msg )
2022-03-22 17:02:59 +01:00
Notification . create (
user_id = alias . user_id ,
title = f " { alias . email } has a new mail in quarantine " ,
message = Notification . render (
" notification/message-quarantine.html " , alias = alias
) ,
commit = True ,
)
2022-03-25 17:25:35 +01:00
user = alias . user
send_email_with_rate_control (
user ,
ALERT_QUARANTINE_DMARC ,
user . email ,
f " An email sent to { alias . email } has been quarantined " ,
render (
" transactional/message-quarantine-dmarc.txt.jinja2 " ,
from_header = from_header ,
alias = alias ,
refused_email_url = email_log . get_dashboard_url ( ) ,
) ,
render (
" transactional/message-quarantine-dmarc.html " ,
from_header = from_header ,
alias = alias ,
refused_email_url = email_log . get_dashboard_url ( ) ,
) ,
max_nb_alert = 10 ,
ignore_smtp_error = True ,
)
2022-03-22 17:02:59 +01:00
return status . E215
2022-03-23 08:34:25 +01:00
2022-03-21 19:05:15 +01:00
return None
2022-03-18 15:44:07 +01:00
2022-03-21 19:05:15 +01:00
2022-03-25 17:25:35 +01:00
def quarantine_dmarc_failed_email ( alias , contact , envelope , msg ) - > EmailLog :
2022-03-21 19:05:15 +01:00
add_or_replace_header ( msg , headers . SL_DIRECTION , " Forward " )
msg [ headers . SL_ENVELOPE_TO ] = alias . email
msg [ headers . SL_ENVELOPE_FROM ] = envelope . mail_from
add_or_replace_header ( msg , " From " , contact . new_addr ( ) )
# replace CC & To emails by reverse-alias for all emails that are not alias
try :
replace_header_when_forward ( msg , alias , " Cc " )
replace_header_when_forward ( msg , alias , " To " )
except CannotCreateContactForReverseAlias :
Session . commit ( )
raise
2022-03-25 17:25:35 +01:00
2022-03-21 19:05:15 +01:00
random_name = str ( uuid . uuid4 ( ) )
s3_report_path = f " refused-emails/full- { random_name } .eml "
s3 . upload_email_from_bytesio (
s3_report_path , BytesIO ( to_bytes ( msg ) ) , f " full- { random_name } "
)
refused_email = RefusedEmail . create (
full_report_path = s3_report_path , user_id = alias . user_id , flush = True
)
2022-03-25 17:25:35 +01:00
return EmailLog . create (
2022-03-21 19:05:15 +01:00
user_id = alias . user_id ,
mailbox_id = alias . mailbox_id ,
contact_id = contact . id ,
alias_id = alias . id ,
message_id = str ( msg [ headers . MESSAGE_ID ] ) ,
refused_email_id = refused_email . id ,
is_spam = True ,
blocked = True ,
commit = True ,
)
2020-09-30 11:05:21 +02:00
def handle_forward ( envelope , msg : Message , rcpt_to : str ) - > List [ Tuple [ bool , str ] ] :
2020-08-30 19:22:21 +02:00
""" return an array of SMTP status (is_success, smtp_status)
is_success indicates whether an email has been delivered and
2022-03-11 08:56:27 +01:00
smtp_status is the SMTP Status ( " 250 Message accepted " , " 550 Non-existent email address " , etc . )
2020-03-28 21:24:43 +01:00
"""
2021-09-10 17:48:36 +02:00
alias_address = rcpt_to # alias@SL
2020-02-19 16:49:40 +01:00
2021-09-10 17:48:36 +02:00
alias = Alias . get_by ( email = alias_address )
2020-03-17 11:51:40 +01:00
if not alias :
2021-09-10 17:48:36 +02:00
LOG . d (
" alias %s not exist. Try to see if it can be created on the fly " ,
alias_address ,
)
alias = try_auto_create ( alias_address )
2020-03-17 11:51:40 +01:00
if not alias :
2021-09-10 17:48:36 +02:00
LOG . d ( " alias %s cannot be created on-the-fly, return 550 " , alias_address )
2021-08-02 11:33:58 +02:00
if should_ignore_bounce ( envelope . mail_from ) :
return [ ( True , status . E207 ) ]
else :
return [ ( False , status . E515 ) ]
2020-02-19 16:49:40 +01:00
2020-11-04 12:32:15 +01:00
user = alias . user
if user . disabled :
2021-03-16 09:17:23 +01:00
LOG . w ( " User %s disabled, disable forwarding emails for %s " , user , alias )
2021-08-02 11:33:58 +02:00
if should_ignore_bounce ( envelope . mail_from ) :
return [ ( True , status . E207 ) ]
else :
return [ ( False , status . E504 ) ]
2020-10-04 12:49:27 +02:00
2022-01-07 16:22:35 +01:00
# check if email is sent from alias's owning mailbox(es)
mail_from = envelope . mail_from
2022-01-08 00:09:45 +01:00
for addr in alias . authorized_addresses ( ) :
2022-01-07 16:22:35 +01:00
# email sent from a mailbox to its alias
2022-01-08 00:09:45 +01:00
if addr == mail_from :
LOG . i ( " cycle email sent from %s to %s " , addr , alias )
handle_email_sent_to_ourself ( alias , addr , msg , user )
2022-01-07 16:22:35 +01:00
return [ ( True , status . E209 ) ]
2020-08-25 12:51:05 +02:00
2021-10-11 12:10:18 +02:00
from_header = get_header_unicode ( msg [ headers . FROM ] )
2021-07-28 09:12:52 +02:00
LOG . d ( " Create or get contact for from_header: %s " , from_header )
2021-12-31 12:14:22 +01:00
try :
2022-01-07 13:02:16 +01:00
contact = get_or_create_contact ( from_header , envelope . mail_from , alias , msg )
2021-12-31 12:14:22 +01:00
except ObjectDeletedError :
LOG . d ( " maybe alias was deleted in the meantime " )
alias = Alias . get_by ( email = alias_address )
if not alias :
LOG . i ( " Alias %s was deleted in the meantime " , alias_address )
if should_ignore_bounce ( envelope . mail_from ) :
return [ ( True , status . E207 ) ]
else :
return [ ( False , status . E515 ) ]
2020-11-04 12:32:15 +01:00
2021-07-28 09:12:52 +02:00
reply_to_contact = None
2021-10-11 12:10:18 +02:00
if msg [ headers . REPLY_TO ] :
reply_to = get_header_unicode ( msg [ headers . REPLY_TO ] )
2022-01-07 17:53:06 +01:00
LOG . d ( " Create or get contact for reply_to_header: %s " , reply_to )
2021-07-28 09:12:52 +02:00
# ignore when reply-to = alias
if reply_to == alias . email :
LOG . i ( " Reply-to same as alias %s " , alias )
else :
2022-01-07 13:02:16 +01:00
reply_to_contact = get_or_create_reply_to_contact ( reply_to , alias , msg )
2021-07-28 09:12:52 +02:00
2021-10-28 10:19:58 +02:00
if not alias . enabled or contact . block_forward :
2020-04-27 18:18:40 +02:00
LOG . d ( " %s is disabled, do not forward " , alias )
2020-11-24 16:38:34 +01:00
EmailLog . create (
2021-07-11 12:28:23 +02:00
contact_id = contact . id ,
user_id = contact . user_id ,
blocked = True ,
alias_id = contact . alias_id ,
commit = True ,
2020-11-24 16:38:34 +01:00
)
2022-02-21 12:52:21 +01:00
res_status = status . E200
if user . block_behaviour == BlockBehaviourEnum . return_5xx :
res_status = status . E502
2021-10-28 10:19:58 +02:00
# do not return 5** to allow user to receive emails later when alias is enabled or contact is unblocked
2022-02-21 12:52:21 +01:00
return [ ( True , res_status ) ]
2020-03-30 22:05:31 +02:00
2022-03-18 15:44:07 +01:00
# Check if we need to reject or quarantine based on dmarc
2022-03-25 17:25:35 +01:00
dmarc_delivery_status = apply_dmarc_policy (
alias , contact , envelope , msg , from_header
)
2022-03-18 15:44:07 +01:00
if dmarc_delivery_status is not None :
return [ ( False , dmarc_delivery_status ) ]
2020-05-10 16:57:47 +02:00
ret = [ ]
2020-08-21 10:41:50 +02:00
mailboxes = alias . mailboxes
2020-08-30 19:22:21 +02:00
# no valid mailbox
if not mailboxes :
2022-01-08 16:59:32 +01:00
LOG . w ( " no valid mailboxes for %s " , alias )
2021-08-02 11:33:58 +02:00
if should_ignore_bounce ( envelope . mail_from ) :
return [ ( True , status . E207 ) ]
else :
return [ ( False , status . E516 ) ]
2020-08-30 19:22:21 +02:00
2020-11-04 14:55:54 +01:00
for mailbox in mailboxes :
2020-09-10 09:38:30 +02:00
if not mailbox . verified :
2021-03-16 09:17:23 +01:00
LOG . d ( " %s unverified, do not forward " , mailbox )
2021-06-23 19:47:06 +02:00
ret . append ( ( False , status . E517 ) )
2020-09-10 09:38:30 +02:00
else :
2020-11-04 14:55:54 +01:00
# create a copy of message for each forward
2020-09-10 09:38:30 +02:00
ret . append (
2020-09-30 11:05:21 +02:00
forward_email_to_mailbox (
2021-07-28 09:12:52 +02:00
alias , copy ( msg ) , contact , envelope , mailbox , user , reply_to_contact
2020-09-10 09:38:30 +02:00
)
2020-05-10 16:57:47 +02:00
)
return ret
2020-09-30 11:05:21 +02:00
def forward_email_to_mailbox (
2020-05-10 16:57:47 +02:00
alias ,
msg : Message ,
contact : Contact ,
envelope ,
mailbox ,
user ,
2021-07-28 09:12:52 +02:00
reply_to_contact : Optional [ Contact ] ,
2020-05-10 16:57:47 +02:00
) - > ( bool , str ) :
LOG . d ( " Forward %s -> %s -> %s " , contact , alias , mailbox )
2020-06-09 17:16:32 +02:00
2020-10-12 13:28:21 +02:00
if mailbox . disabled :
2021-08-02 11:33:58 +02:00
LOG . d ( " %s disabled, do not forward " )
if should_ignore_bounce ( envelope . mail_from ) :
return True , status . E207
else :
return False , status . E518
2020-10-12 13:28:21 +02:00
2020-06-09 17:16:32 +02:00
# sanity check: make sure mailbox is not actually an alias
if get_email_domain_part ( alias . email ) == get_email_domain_part ( mailbox . email ) :
2021-03-16 09:17:23 +01:00
LOG . w (
2020-06-09 17:16:32 +02:00
" Mailbox has the same domain as alias. %s -> %s -> %s " ,
contact ,
alias ,
mailbox ,
)
2020-08-30 19:06:50 +02:00
mailbox_url = f " { URL } /dashboard/mailbox/ { mailbox . id } / "
send_email_with_rate_control (
user ,
ALERT_MAILBOX_IS_ALIAS ,
user . email ,
2020-12-21 09:39:26 +01:00
f " Your mailbox { mailbox . email } and alias { alias . email } use the same domain " ,
2020-08-30 19:06:50 +02:00
render (
2021-11-02 11:37:46 +01:00
" transactional/mailbox-invalid.txt.jinja2 " ,
2020-08-30 19:06:50 +02:00
mailbox = mailbox ,
mailbox_url = mailbox_url ,
2020-12-21 11:57:12 +01:00
alias = alias ,
2020-08-30 19:06:50 +02:00
) ,
render (
" transactional/mailbox-invalid.html " ,
mailbox = mailbox ,
mailbox_url = mailbox_url ,
2020-12-21 11:57:12 +01:00
alias = alias ,
2020-08-30 19:06:50 +02:00
) ,
2020-08-31 17:32:46 +02:00
max_nb_alert = 1 ,
2020-08-30 19:06:50 +02:00
)
2020-08-30 19:08:53 +02:00
# retry later
# so when user fixes the mailbox, the email can be delivered
2021-06-23 19:47:06 +02:00
return False , status . E405
2020-06-09 17:16:32 +02:00
2020-11-24 16:38:34 +01:00
email_log = EmailLog . create (
2021-07-11 12:28:23 +02:00
contact_id = contact . id ,
user_id = user . id ,
mailbox_id = mailbox . id ,
alias_id = contact . alias_id ,
2021-11-10 09:38:20 +01:00
message_id = str ( msg [ headers . MESSAGE_ID ] ) ,
2021-07-11 12:28:23 +02:00
commit = True ,
2020-11-24 16:38:34 +01:00
)
2021-05-24 12:08:30 +02:00
LOG . d ( " Create %s for %s , %s , %s " , email_log , contact , user , mailbox )
2020-11-24 16:38:34 +01:00
2021-03-26 10:00:48 +01:00
if ENABLE_SPAM_ASSASSIN :
# Spam check
spam_status = " "
is_spam = False
2020-08-15 16:38:16 +02:00
2021-03-26 10:00:48 +01:00
if SPAMASSASSIN_HOST :
start = time . time ( )
spam_score , spam_report = get_spam_score ( msg , email_log )
LOG . d (
" %s -> %s - spam score: %s in %s seconds. Spam report %s " ,
contact ,
alias ,
spam_score ,
time . time ( ) - start ,
spam_report ,
)
email_log . spam_score = spam_score
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-08-20 11:58:46 +02:00
2021-03-26 10:00:48 +01:00
if ( user . max_spam_score and spam_score > user . max_spam_score ) or (
not user . max_spam_score and spam_score > MAX_SPAM_SCORE
) :
is_spam = True
# only set the spam report for spam
email_log . spam_report = spam_report
else :
is_spam , spam_status = get_spam_info ( msg , max_score = user . max_spam_score )
2020-08-15 16:38:16 +02:00
2021-03-26 10:00:48 +01:00
if is_spam :
LOG . w (
" Email detected as spam. %s -> %s . Spam Score: %s , Spam Report: %s " ,
contact ,
alias ,
email_log . spam_score ,
email_log . spam_report ,
)
email_log . is_spam = True
email_log . spam_status = spam_status
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-05-15 16:34:07 +02:00
2021-03-26 10:00:48 +01:00
handle_spam ( contact , alias , msg , user , mailbox , email_log )
2021-06-23 19:47:06 +02:00
return False , status . E519
2020-03-30 22:05:31 +02:00
2020-11-14 15:55:53 +01:00
if contact . invalid_email :
LOG . d ( " add noreply information %s %s " , alias , mailbox )
msg = add_header (
msg ,
f """ Email sent to { alias . email } from an invalid address and cannot be replied """ ,
f """ Email sent to { alias . email } from an invalid address and cannot be replied """ ,
)
2021-01-28 16:34:24 +01:00
delete_all_headers_except (
msg ,
[
2021-10-11 12:19:21 +02:00
headers . FROM ,
headers . TO ,
headers . CC ,
headers . SUBJECT ,
2022-01-07 10:22:46 +01:00
headers . DATE ,
2021-10-11 12:00:37 +02:00
# do not delete original message id
headers . MESSAGE_ID ,
2021-01-28 16:34:24 +01:00
# References and In-Reply-To are used for keeping the email thread
2021-10-11 12:00:37 +02:00
headers . REFERENCES ,
headers . IN_REPLY_TO ,
2021-01-28 16:34:24 +01:00
]
2021-10-11 12:19:21 +02:00
+ headers . MIME_HEADERS ,
2021-01-28 16:34:24 +01:00
)
2020-03-08 23:07:23 +01:00
# create PGP email if needed
2020-11-24 11:28:14 +01:00
if mailbox . pgp_enabled ( ) and user . is_premium ( ) and not alias . disable_pgp :
2020-03-08 23:07:23 +01:00
LOG . d ( " Encrypt message using mailbox %s " , mailbox )
2020-11-07 13:00:45 +01:00
if mailbox . generic_subject :
LOG . d ( " Use a generic subject for %s " , mailbox )
2021-10-11 12:10:18 +02:00
orig_subject = msg [ headers . SUBJECT ]
2020-11-09 21:16:50 +01:00
orig_subject = get_header_unicode ( orig_subject )
2020-11-07 13:00:45 +01:00
add_or_replace_header ( msg , " Subject " , mailbox . generic_subject )
msg = add_header (
msg ,
2020-11-07 17:23:28 +01:00
f """ Forwarded by SimpleLogin to { alias . email } with " { orig_subject } " as subject """ ,
f """ Forwarded by SimpleLogin to { alias . email } with <b> { orig_subject } </b> as subject """ ,
2020-11-07 13:00:45 +01:00
)
2020-06-08 13:54:42 +02:00
try :
2020-10-28 11:50:14 +01:00
msg = prepare_pgp_message (
2020-11-02 19:09:57 +01:00
msg , mailbox . pgp_finger_print , mailbox . pgp_public_key , can_sign = True
2020-10-28 11:50:14 +01:00
)
2020-06-08 13:54:42 +02:00
except PGPException :
2021-06-23 19:47:06 +02:00
LOG . e (
2020-06-08 13:54:42 +02:00
" Cannot encrypt message %s -> %s . %s %s " , contact , alias , mailbox , user
)
2022-01-08 16:59:32 +01:00
EmailLog . delete ( email_log . id , commit = True )
2020-06-08 13:54:42 +02:00
# so the client can retry later
2021-06-23 19:47:06 +02:00
return False , status . E406
2020-03-08 23:07:23 +01:00
2020-04-27 18:18:40 +02:00
# add custom header
2021-10-11 12:19:21 +02:00
add_or_replace_header ( msg , headers . SL_DIRECTION , " Forward " )
2020-04-27 18:18:40 +02:00
2021-10-11 12:19:21 +02:00
msg [ headers . SL_EMAIL_LOG_ID ] = str ( email_log . id )
2022-02-25 12:22:09 +01:00
if user . include_header_email_header :
msg [ headers . SL_ENVELOPE_FROM ] = envelope . mail_from
2021-05-12 10:46:07 +02:00
# when an alias isn't in the To: header, there's no way for users to know what alias has received the email
2021-10-11 12:19:21 +02:00
msg [ headers . SL_ENVELOPE_TO ] = alias . email
2021-10-11 12:00:37 +02:00
if not msg [ headers . DATE ] :
LOG . w ( " missing date header, create one " )
msg [ headers . DATE ] = formatdate ( )
2020-11-02 14:53:22 +01:00
2021-11-01 18:45:10 +01:00
replace_sl_message_id_by_original_message_id ( msg )
2021-10-18 17:35:16 +02:00
2022-01-08 16:59:32 +01:00
# change the from_header so the email comes from a reverse-alias
2020-04-27 18:18:40 +02:00
# replace the email part in from: header
2022-01-08 16:59:32 +01:00
old_from_header = msg [ headers . FROM ]
2020-04-27 18:18:40 +02:00
new_from_header = contact . new_addr ( )
add_or_replace_header ( msg , " From " , new_from_header )
2022-01-08 16:59:32 +01:00
LOG . d ( " From header, new: %s , old: %s " , new_from_header , old_from_header )
2020-04-27 18:18:40 +02:00
2021-07-28 09:12:52 +02:00
if reply_to_contact :
2021-10-11 12:10:18 +02:00
reply_to_header = msg [ headers . REPLY_TO ]
2021-07-28 09:12:52 +02:00
new_reply_to_header = reply_to_contact . new_addr ( )
add_or_replace_header ( msg , " Reply-To " , new_reply_to_header )
LOG . d ( " Reply-To header, new: %s , old: %s " , new_reply_to_header , reply_to_header )
2020-11-01 18:13:50 +01:00
# replace CC & To emails by reverse-alias for all emails that are not alias
2022-01-08 00:16:32 +01:00
try :
replace_header_when_forward ( msg , alias , " Cc " )
replace_header_when_forward ( msg , alias , " To " )
except CannotCreateContactForReverseAlias :
LOG . d ( " CannotCreateContactForReverseAlias error, delete %s " , email_log )
EmailLog . delete ( email_log . id )
Session . commit ( )
raise
2020-04-27 18:18:40 +02:00
# add List-Unsubscribe header
2021-11-02 15:44:43 +01:00
if user . one_click_unsubscribe_block_sender :
unsubscribe_link , via_email = alias . unsubscribe_link ( contact )
else :
unsubscribe_link , via_email = alias . unsubscribe_link ( )
2021-11-02 14:36:37 +01:00
add_or_replace_header ( msg , headers . LIST_UNSUBSCRIBE , f " < { unsubscribe_link } > " )
2020-10-22 10:37:02 +02:00
if not via_email :
2020-04-27 18:18:40 +02:00
add_or_replace_header (
2021-11-02 14:36:37 +01:00
msg , headers . LIST_UNSUBSCRIBE_POST , " List-Unsubscribe=One-Click "
2019-11-19 10:23:06 +01:00
)
2019-12-15 10:18:33 +01:00
2021-09-25 18:47:15 +02:00
add_dkim_signature ( msg , EMAIL_DOMAIN )
2020-04-27 18:18:40 +02:00
LOG . d (
2021-03-16 09:17:23 +01:00
" Forward mail from %s to %s , mail_options: %s , rcpt_options: %s " ,
2020-04-27 18:18:40 +02:00
contact . website_email ,
2020-05-10 16:32:54 +02:00
mailbox . email ,
2020-04-27 18:18:40 +02:00
envelope . mail_options ,
envelope . rcpt_options ,
)
2020-09-02 10:16:13 +02:00
try :
2020-09-29 12:57:14 +02:00
sl_sendmail (
2021-01-11 14:55:55 +01:00
# use a different envelope sender for each forward (aka VERP)
BOUNCE_EMAIL . format ( email_log . id ) ,
2020-09-02 10:16:13 +02:00
mailbox . email ,
2020-09-29 12:57:14 +02:00
msg ,
2020-09-02 10:16:13 +02:00
envelope . mail_options ,
envelope . rcpt_options ,
2021-01-28 13:50:24 +01:00
is_forward = True ,
2020-09-02 10:16:13 +02:00
)
2021-12-29 15:17:57 +01:00
except ( SMTPServerDisconnected , SMTPRecipientsRefused , TimeoutError ) :
2021-03-16 09:17:23 +01:00
LOG . w (
2021-12-29 15:17:57 +01:00
" Postfix error during forward phase %s -> %s -> %s " ,
2020-09-02 10:16:13 +02:00
contact ,
alias ,
mailbox ,
2021-12-23 19:34:17 +01:00
exc_info = True ,
2020-09-02 10:16:13 +02:00
)
2021-08-02 11:33:58 +02:00
if should_ignore_bounce ( envelope . mail_from ) :
return True , status . E207
else :
2022-01-08 16:59:32 +01:00
EmailLog . delete ( email_log . id , commit = True )
2021-12-29 15:17:57 +01:00
# so Postfix can retry
2021-12-23 18:17:29 +01:00
return False , status . E407
2020-09-02 10:16:13 +02:00
else :
2021-10-12 14:36:47 +02:00
Session . commit ( )
2021-06-23 19:47:06 +02:00
return True , status . E200
2019-11-19 10:23:06 +01:00
2021-11-01 18:45:10 +01:00
def replace_sl_message_id_by_original_message_id ( msg ) :
# Replace SL Message-ID by original one in In-Reply-To header
if msg [ headers . IN_REPLY_TO ] :
matching : MessageIDMatching = MessageIDMatching . get_by (
2021-11-08 11:21:01 +01:00
sl_message_id = str ( msg [ headers . IN_REPLY_TO ] )
2021-11-01 18:45:10 +01:00
)
if matching :
LOG . d (
" replace SL message id by original one in in-reply-to header, %s -> %s " ,
msg [ headers . IN_REPLY_TO ] ,
matching . original_message_id ,
)
del msg [ headers . IN_REPLY_TO ]
msg [ headers . IN_REPLY_TO ] = matching . original_message_id
# Replace SL Message-ID by original Message-ID in References header
if msg [ headers . REFERENCES ] :
message_ids = msg [ headers . REFERENCES ] . split ( )
new_message_ids = [ ]
for message_id in message_ids :
matching = MessageIDMatching . get_by ( sl_message_id = message_id )
if matching :
LOG . d (
" replace SL message id by original one in references header, %s -> %s " ,
message_id ,
matching . original_message_id ,
)
new_message_ids . append ( matching . original_message_id )
else :
new_message_ids . append ( message_id )
del msg [ headers . REFERENCES ]
msg [ headers . REFERENCES ] = " " . join ( new_message_ids )
2020-09-30 11:05:21 +02:00
def handle_reply ( envelope , msg : Message , rcpt_to : str ) - > ( bool , str ) :
2020-03-28 21:24:43 +01:00
"""
2021-01-28 16:34:24 +01:00
Return whether an email has been delivered and
2020-03-28 21:24:43 +01:00
the smtp status ( " 250 Message accepted " , " 550 Non-existent email address " , etc )
"""
2020-09-14 17:38:48 +02:00
reply_email = rcpt_to
2019-11-19 10:23:06 +01:00
2020-02-19 16:17:13 +01:00
# reply_email must end with EMAIL_DOMAIN
if not reply_email . endswith ( EMAIL_DOMAIN ) :
2021-03-16 09:17:23 +01:00
LOG . w ( f " Reply email { reply_email } has wrong domain " )
2021-06-23 19:47:06 +02:00
return False , status . E501
2019-11-19 10:23:06 +01:00
2020-11-22 13:07:09 +01:00
# handle case where reply email is generated with non-allowed char
reply_email = normalize_reply_email ( reply_email )
2020-11-18 16:11:00 +01:00
2020-03-17 10:56:59 +01:00
contact = Contact . get_by ( reply_email = reply_email )
if not contact :
2022-03-11 08:56:27 +01:00
LOG . w ( f " No contact with { reply_email } as reverse alias " )
2021-06-23 19:47:06 +02:00
return False , status . E502
2019-12-18 17:07:20 +01:00
2020-03-28 19:16:55 +01:00
alias = contact . alias
2021-12-30 10:24:57 +01:00
alias_address : str = contact . alias . email
alias_domain = alias_address [ alias_address . find ( " @ " ) + 1 : ]
2019-11-19 10:23:06 +01:00
2020-10-15 16:21:31 +02:00
# Sanity check: verify alias domain is managed by SimpleLogin
# scenario: a user have removed a domain but due to a bug, the aliases are still there
if not is_valid_alias_address_domain ( alias . email ) :
2021-09-08 11:29:55 +02:00
LOG . e ( " %s domain isn ' t known " , alias )
2021-06-23 19:47:06 +02:00
return False , status . E503
2019-11-19 10:23:06 +01:00
2020-03-17 11:51:40 +01:00
user = alias . user
2020-09-14 17:38:48 +02:00
mail_from = envelope . mail_from
2020-02-19 16:17:13 +01:00
2020-10-04 12:49:27 +02:00
if user . disabled :
2021-09-08 11:29:55 +02:00
LOG . e (
2020-10-04 12:49:27 +02:00
" User %s disabled, disable sending emails from %s to %s " ,
user ,
alias ,
contact ,
)
2021-06-23 19:47:06 +02:00
return [ ( False , status . E504 ) ]
2020-10-04 12:49:27 +02:00
2020-09-28 17:43:09 +02:00
# Anti-spoofing
mailbox = get_mailbox_from_mail_from ( mail_from , alias )
if not mailbox :
2020-08-26 14:39:51 +02:00
if alias . disable_email_spoofing_check :
# ignore this error, use default alias mailbox
2021-03-16 09:17:23 +01:00
LOG . w (
2020-08-26 14:39:51 +02:00
" ignore unknown sender to reverse-alias %s : %s -> %s " ,
mail_from ,
alias ,
contact ,
)
mailbox = alias . mailbox
else :
# only mailbox can send email to the reply-email
2020-09-16 17:24:42 +02:00
handle_unknown_mailbox ( envelope , msg , reply_email , user , alias , contact )
2022-03-16 09:06:49 +01:00
# return 2** to avoid Postfix sending out bounces and avoid backscatter issue
return False , status . E214
2020-05-10 18:19:29 +02:00
2020-08-26 14:39:51 +02:00
if ENFORCE_SPF and mailbox . force_spf and not alias . disable_email_spoofing_check :
2021-03-17 10:16:09 +01:00
if not spf_pass ( envelope , mailbox , user , alias , contact . website_email , msg ) :
2021-06-23 19:47:06 +02:00
# cannot use 4** here as sender will retry.
# cannot use 5** because that generates bounce report
return True , status . E201
2020-05-07 13:28:04 +02:00
2020-08-16 14:28:47 +02:00
email_log = EmailLog . create (
2020-11-24 16:38:34 +01:00
contact_id = contact . id ,
2021-07-11 12:28:23 +02:00
alias_id = contact . alias_id ,
2020-11-24 16:38:34 +01:00
is_reply = True ,
user_id = contact . user_id ,
mailbox_id = mailbox . id ,
2021-10-18 17:35:16 +02:00
message_id = msg [ headers . MESSAGE_ID ] ,
2020-12-11 11:13:19 +01:00
commit = True ,
2020-08-16 14:28:47 +02:00
)
2021-05-24 12:08:30 +02:00
LOG . d ( " Create %s for %s , %s , %s " , email_log , contact , user , mailbox )
2020-08-16 14:28:47 +02:00
2020-08-15 16:53:57 +02:00
# Spam check
2021-03-26 10:00:48 +01:00
if ENABLE_SPAM_ASSASSIN :
spam_status = " "
is_spam = False
# do not use user.max_spam_score here
if SPAMASSASSIN_HOST :
start = time . time ( )
spam_score , spam_report = get_spam_score ( msg , email_log )
LOG . d (
" %s -> %s - spam score %s in %s seconds. Spam report %s " ,
alias ,
contact ,
spam_score ,
time . time ( ) - start ,
spam_report ,
)
email_log . spam_score = spam_score
if spam_score > MAX_REPLY_PHASE_SPAM_SCORE :
is_spam = True
# only set the spam report for spam
email_log . spam_report = spam_report
else :
is_spam , spam_status = get_spam_info (
msg , max_score = MAX_REPLY_PHASE_SPAM_SCORE
)
2020-08-15 16:53:57 +02:00
2021-03-26 10:00:48 +01:00
if is_spam :
LOG . w (
" Email detected as spam. Reply phase. %s -> %s . Spam Score: %s , Spam Report: %s " ,
alias ,
contact ,
email_log . spam_score ,
email_log . spam_report ,
)
2020-08-16 14:28:47 +02:00
2021-03-26 10:00:48 +01:00
email_log . is_spam = True
email_log . spam_status = spam_status
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-08-15 16:53:57 +02:00
2021-03-26 10:00:48 +01:00
handle_spam ( contact , alias , msg , user , mailbox , email_log , is_reply = True )
2021-06-23 19:47:06 +02:00
return False , status . E506
2020-08-15 16:53:57 +02:00
2020-11-02 14:51:37 +01:00
delete_all_headers_except (
msg ,
[
2021-10-11 12:19:21 +02:00
headers . FROM ,
headers . TO ,
headers . CC ,
headers . SUBJECT ,
2022-01-07 10:22:46 +01:00
headers . DATE ,
2021-10-11 12:00:37 +02:00
# do not delete original message id
headers . MESSAGE_ID ,
2020-12-11 11:13:19 +01:00
# References and In-Reply-To are used for keeping the email thread
2021-10-11 12:00:37 +02:00
headers . REFERENCES ,
headers . IN_REPLY_TO ,
2020-11-02 14:51:37 +01:00
]
2021-10-11 12:19:21 +02:00
+ headers . MIME_HEADERS ,
2020-11-02 14:51:37 +01:00
)
2020-11-01 18:12:09 +01:00
2021-10-25 14:33:42 +02:00
# replace the reverse-alias by the contact email in the email body
2020-11-01 18:02:43 +01:00
# as this is usually included when replying
if user . replace_reverse_alias :
2020-11-30 15:15:44 +01:00
LOG . d ( " Replace reverse-alias %s by contact email %s " , reply_email , contact )
msg = replace ( msg , reply_email , contact . website_email )
2020-11-01 18:02:43 +01:00
# create PGP email if needed
if contact . pgp_finger_print and user . is_premium ( ) :
LOG . d ( " Encrypt message for contact %s " , contact )
try :
msg = prepare_pgp_message (
msg , contact . pgp_finger_print , contact . pgp_public_key
)
except PGPException :
2021-06-23 19:55:41 +02:00
LOG . e (
2020-11-01 18:02:43 +01:00
" Cannot encrypt message %s -> %s . %s %s " , alias , contact , mailbox , user
)
2021-11-02 10:59:17 +01:00
# programming error, user shouldn't see a new email log
2022-01-08 00:28:26 +01:00
EmailLog . delete ( email_log . id , commit = True )
2020-11-01 18:02:43 +01:00
# return 421 so the client can retry later
2021-06-23 19:47:06 +02:00
return False , status . E402
2020-11-01 18:02:43 +01:00
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-11-04 12:32:15 +01:00
2020-03-28 19:16:55 +01:00
# make the email comes from alias
2020-05-03 16:05:34 +02:00
from_header = alias . email
# add alias name from alias
2020-04-26 10:41:24 +02:00
if alias . name :
2021-11-22 11:23:21 +01:00
LOG . d ( " Put alias name %s in from header " , alias . name )
2020-04-26 10:41:24 +02:00
from_header = formataddr ( ( alias . name , alias . email ) )
2020-05-03 16:05:34 +02:00
elif alias . custom_domain :
# add alias name from domain
if alias . custom_domain . name :
2021-11-22 11:23:21 +01:00
LOG . d (
" Put domain default alias name %s in from header " ,
alias . custom_domain . name ,
)
2020-05-03 16:05:34 +02:00
from_header = formataddr ( ( alias . custom_domain . name , alias . email ) )
2021-11-22 11:23:21 +01:00
LOG . d ( " From header is %s " , from_header )
2021-10-11 12:13:24 +02:00
add_or_replace_header ( msg , headers . FROM , from_header )
2020-02-10 17:24:14 +01:00
2022-01-07 17:53:06 +01:00
try :
replace_header_when_reply ( msg , alias , headers . TO )
replace_header_when_reply ( msg , alias , headers . CC )
except NonReverseAliasInReplyPhase as e :
LOG . w ( " non reverse-alias in reply %s %s %s " , e , contact , alias )
2022-01-08 00:28:26 +01:00
# the email is ignored, delete the email log
EmailLog . delete ( email_log . id , commit = True )
2022-01-11 13:14:47 +01:00
send_email (
2022-01-07 17:53:06 +01:00
mailbox . email ,
f " Email sent to { contact . email } contains non reverse-alias addresses " ,
render (
" transactional/non-reverse-alias-reply-phase.txt.jinja2 " ,
destination = contact . email ,
alias = alias . email ,
subject = msg [ headers . SUBJECT ] ,
) ,
)
# user is informed and will retry
return True , status . E200
2019-12-15 17:04:46 +01:00
2021-11-01 18:43:19 +01:00
replace_original_message_id ( alias , email_log , msg )
2021-10-11 12:13:24 +02:00
if not msg [ headers . DATE ] :
date_header = formatdate ( )
LOG . w ( " missing date header, add one " )
msg [ headers . DATE ] = date_header
2020-08-25 12:47:28 +02:00
2021-10-11 12:19:21 +02:00
msg [ headers . SL_DIRECTION ] = " Reply "
msg [ headers . SL_EMAIL_LOG_ID ] = str ( email_log . id )
2020-08-25 12:47:28 +02:00
2020-02-19 16:17:13 +01:00
LOG . d (
" send email from %s to %s , mail_options: %s ,rcpt_options: %s " ,
2020-03-28 19:16:55 +01:00
alias . email ,
2020-03-17 10:56:59 +01:00
contact . website_email ,
2020-02-19 16:17:13 +01:00
envelope . mail_options ,
envelope . rcpt_options ,
)
2019-12-17 17:48:06 +01:00
2021-09-25 18:47:15 +02:00
if should_add_dkim_signature ( alias_domain ) :
add_dkim_signature ( msg , alias_domain )
2021-05-25 17:58:45 +02:00
# generate a mail_from for VERP
verp_mail_from = f " { BOUNCE_PREFIX_FOR_REPLY_PHASE } + { email_log . id } +@ { alias_domain } "
2020-06-20 16:19:01 +02:00
try :
2020-09-29 12:57:14 +02:00
sl_sendmail (
2021-05-25 17:58:45 +02:00
verp_mail_from ,
2020-06-20 16:19:01 +02:00
contact . website_email ,
2020-09-29 12:57:14 +02:00
msg ,
2020-06-20 16:19:01 +02:00
envelope . mail_options ,
envelope . rcpt_options ,
2021-01-28 13:50:24 +01:00
is_forward = False ,
2020-06-20 16:19:01 +02:00
)
except Exception :
2021-03-16 09:17:23 +01:00
LOG . w ( " Cannot send email from %s to %s " , alias , contact )
2022-01-08 16:59:32 +01:00
EmailLog . delete ( email_log . id , commit = True )
2020-06-20 16:19:01 +02:00
send_email (
mailbox . email ,
f " Email cannot be sent to { contact . email } from { alias . email } " ,
render (
2021-11-02 10:58:51 +01:00
" transactional/reply-error.txt.jinja2 " ,
2020-06-20 16:19:01 +02:00
user = user ,
alias = alias ,
contact = contact ,
contact_domain = get_email_domain_part ( contact . email ) ,
) ,
render (
" transactional/reply-error.html " ,
user = user ,
alias = alias ,
contact = contact ,
contact_domain = get_email_domain_part ( contact . email ) ,
) ,
)
2020-01-07 19:14:36 +01:00
2020-09-02 10:16:13 +02:00
# return 250 even if error as user is already informed of the incident and can retry sending the email
2021-06-23 19:47:06 +02:00
return True , status . E200
2019-11-20 18:52:49 +01:00
2020-01-08 12:44:29 +01:00
2021-11-01 18:43:19 +01:00
def replace_original_message_id ( alias : Alias , email_log : EmailLog , msg : Message ) :
"""
Replace original Message - ID by SL - Message - ID during the reply phase
for " message-id " and " References " headers
"""
original_message_id = msg [ headers . MESSAGE_ID ]
2021-11-05 09:43:58 +01:00
if original_message_id :
matching = MessageIDMatching . get_by ( original_message_id = original_message_id )
# can happen when a user replies to multiple recipient from their alias
# a SL Message_id will be created for the first recipient
# it should be reused for other recipients
if matching :
sl_message_id = matching . sl_message_id
LOG . d ( " reuse the sl_message_id %s " , sl_message_id )
else :
sl_message_id = make_msgid (
str ( email_log . id ) , get_email_domain_part ( alias . email )
)
LOG . d ( " create a new sl_message_id %s " , sl_message_id )
try :
MessageIDMatching . create (
sl_message_id = sl_message_id ,
original_message_id = original_message_id ,
email_log_id = email_log . id ,
commit = True ,
)
except IntegrityError :
LOG . w (
" another matching with original_message_id %s was created in the mean time " ,
original_message_id ,
)
Session . rollback ( )
matching = MessageIDMatching . get_by (
original_message_id = original_message_id
)
sl_message_id = matching . sl_message_id
2021-11-01 18:43:19 +01:00
else :
sl_message_id = make_msgid (
str ( email_log . id ) , get_email_domain_part ( alias . email )
)
2021-11-05 09:43:58 +01:00
LOG . d ( " no original_message_id, create a new sl_message_id %s " , sl_message_id )
2021-11-01 20:36:15 +01:00
2021-11-01 18:43:19 +01:00
del msg [ headers . MESSAGE_ID ]
msg [ headers . MESSAGE_ID ] = sl_message_id
2021-11-01 20:36:15 +01:00
2021-11-01 18:43:19 +01:00
email_log . sl_message_id = sl_message_id
Session . commit ( )
# Replace all original headers in References header by SL Message ID header if needed
if msg [ headers . REFERENCES ] :
message_ids = msg [ headers . REFERENCES ] . split ( )
new_message_ids = [ ]
for message_id in message_ids :
matching = MessageIDMatching . get_by ( original_message_id = message_id )
if matching :
LOG . d (
" replace original message id by SL one, %s -> %s " ,
message_id ,
matching . sl_message_id ,
)
new_message_ids . append ( matching . sl_message_id )
else :
new_message_ids . append ( message_id )
del msg [ headers . REFERENCES ]
msg [ headers . REFERENCES ] = " " . join ( new_message_ids )
2020-09-28 17:41:16 +02:00
def get_mailbox_from_mail_from ( mail_from : str , alias ) - > Optional [ Mailbox ] :
""" return the corresponding mailbox given the mail_from and alias
Usually the mail_from = mailbox . email but it can also be one of the authorized address
"""
for mailbox in alias . mailboxes :
if mailbox . email == mail_from :
return mailbox
2021-11-22 11:23:21 +01:00
for authorized_address in mailbox . authorized_addresses :
if authorized_address . email == mail_from :
2021-09-08 11:29:55 +02:00
LOG . d (
2021-11-22 11:23:21 +01:00
" Found an authorized address for %s %s %s " ,
alias ,
mailbox ,
authorized_address ,
2020-09-28 17:41:16 +02:00
)
return mailbox
return None
2020-09-16 17:24:42 +02:00
def handle_unknown_mailbox (
envelope , msg , reply_email : str , user : User , alias : Alias , contact : Contact
) :
2021-03-16 09:17:23 +01:00
LOG . w (
2020-12-06 13:54:59 +01:00
" Reply email can only be used by mailbox. "
" Actual mail_from: %s . msg from header: %s , reverse-alias %s , %s %s %s " ,
2020-05-09 23:12:30 +02:00
envelope . mail_from ,
2021-10-11 12:10:18 +02:00
msg [ headers . FROM ] ,
2020-05-09 23:12:30 +02:00
reply_email ,
2020-05-10 18:19:29 +02:00
alias ,
user ,
2020-09-16 17:24:42 +02:00
contact ,
2020-05-09 23:12:30 +02:00
)
2020-09-29 11:00:50 +02:00
authorize_address_link = (
f " { URL } /dashboard/mailbox/ { alias . mailbox_id } /#authorized-address "
)
2020-09-29 13:03:15 +02:00
mailbox_emails = [ mailbox . email for mailbox in alias . mailboxes ]
2020-05-09 23:12:30 +02:00
send_email_with_rate_control (
user ,
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX ,
2020-05-10 18:19:29 +02:00
user . email ,
2020-08-27 11:12:48 +02:00
f " Attempt to use your alias { alias . email } from { envelope . mail_from } " ,
2020-05-09 23:12:30 +02:00
render (
" transactional/reply-must-use-personal-email.txt " ,
2020-05-10 18:19:29 +02:00
alias = alias ,
2020-05-09 23:12:30 +02:00
sender = envelope . mail_from ,
2020-09-29 11:00:50 +02:00
authorize_address_link = authorize_address_link ,
2020-09-29 13:11:04 +02:00
mailbox_emails = mailbox_emails ,
2020-05-09 23:12:30 +02:00
) ,
render (
" transactional/reply-must-use-personal-email.html " ,
2020-05-10 18:19:29 +02:00
alias = alias ,
2020-05-09 23:12:30 +02:00
sender = envelope . mail_from ,
2020-09-29 11:00:50 +02:00
authorize_address_link = authorize_address_link ,
2020-09-29 13:11:04 +02:00
mailbox_emails = mailbox_emails ,
2020-05-09 23:12:30 +02:00
) ,
)
2021-01-11 14:55:55 +01:00
def handle_bounce_forward_phase ( msg : Message , email_log : EmailLog ) :
2020-11-04 12:32:15 +01:00
"""
2021-01-11 14:55:55 +01:00
Handle forward phase bounce
Happens when an email cannot be sent to a mailbox
2020-11-04 12:32:15 +01:00
"""
2021-01-11 14:55:55 +01:00
contact = email_log . contact
alias = contact . alias
2021-01-04 14:22:07 +01:00
user = alias . user
2021-01-11 15:53:08 +01:00
mailbox = email_log . mailbox
# email_log.mailbox should be set during the forward phase
if not mailbox :
2021-09-08 11:29:55 +02:00
LOG . e ( " Use %s default mailbox %s " , alias , alias . mailbox )
2021-01-11 15:53:08 +01:00
mailbox = alias . mailbox
2021-10-14 15:46:52 +02:00
bounce_info = get_mailbox_bounce_info ( msg )
if bounce_info :
2021-10-19 12:05:35 +02:00
Bounce . create (
email = mailbox . email , info = bounce_info . as_bytes ( ) . decode ( ) , commit = True
)
2021-10-14 15:46:52 +02:00
else :
2021-11-02 14:32:16 +01:00
LOG . w ( " cannot get bounce info, debug at %s " , save_email_for_debugging ( msg ) )
2021-10-14 15:46:52 +02:00
Bounce . create ( email = mailbox . email , commit = True )
2021-01-26 09:56:13 +01:00
2021-09-08 11:29:55 +02:00
LOG . d (
2021-01-11 15:53:08 +01:00
" Handle forward bounce %s -> %s -> %s . %s " , contact , alias , mailbox , email_log
)
2021-01-04 14:22:07 +01:00
2021-01-11 14:55:55 +01:00
# Store the bounced email, generate a name for the email
random_name = str ( uuid . uuid4 ( ) )
2020-11-04 15:38:26 +01:00
2021-01-11 14:55:55 +01:00
full_report_path = f " refused-emails/full- { random_name } .eml "
2021-01-11 15:25:54 +01:00
s3 . upload_email_from_bytesio (
full_report_path , BytesIO ( to_bytes ( msg ) ) , f " full- { random_name } "
)
2021-01-11 14:55:55 +01:00
file_path = None
orig_msg = get_orig_message_from_bounce ( msg )
if not orig_msg :
# Some MTA does not return the original message in bounce message
# nothing we can do here
2021-03-16 09:17:23 +01:00
LOG . w (
2021-01-11 14:55:55 +01:00
" Cannot parse original message from bounce message %s %s %s %s " ,
2020-11-04 15:38:26 +01:00
alias ,
user ,
2021-01-11 14:55:55 +01:00
contact ,
full_report_path ,
)
else :
file_path = f " refused-emails/ { random_name } .eml "
2021-01-11 15:25:54 +01:00
s3 . upload_email_from_bytesio (
file_path , BytesIO ( to_bytes ( orig_msg ) ) , random_name
)
2021-01-11 14:55:55 +01:00
refused_email = RefusedEmail . create (
path = file_path , full_report_path = full_report_path , user_id = user . id
)
2021-10-12 14:36:47 +02:00
Session . flush ( )
2021-01-11 14:55:55 +01:00
LOG . d ( " Create refused email %s " , refused_email )
email_log . bounced = True
email_log . refused_email_id = refused_email . id
email_log . bounced_mailbox_id = mailbox . id
2021-10-12 14:36:47 +02:00
Session . commit ( )
2021-01-11 14:55:55 +01:00
refused_email_url = f " { URL } /dashboard/refused_email?highlight_id= { email_log . id } "
2022-03-07 15:44:27 +01:00
alias_will_be_disabled , reason = should_disable ( alias )
2022-03-07 15:45:36 +01:00
if alias_will_be_disabled :
LOG . w (
f " Disable alias { alias } because { reason } . { alias . mailboxes } { alias . user } . Last contact { contact } "
)
alias . enabled = False
2022-03-09 17:59:02 +01:00
Notification . create (
user_id = user . id ,
title = f " { alias . email } has been disabled due to multiple bounces " ,
message = Notification . render (
" notification/alias-disable.html " , alias = alias , mailbox = mailbox
) ,
)
2022-03-10 08:33:26 +01:00
Session . commit ( )
2022-03-07 15:45:36 +01:00
send_email_with_rate_control (
user ,
ALERT_BOUNCE_EMAIL ,
user . email ,
f " Alias { alias . email } has been disabled due to multiple bounces " ,
render (
" transactional/bounce/automatic-disable-alias.txt " ,
alias = alias ,
refused_email_url = refused_email_url ,
mailbox_email = mailbox . email ,
) ,
render (
" transactional/bounce/automatic-disable-alias.html " ,
alias = alias ,
refused_email_url = refused_email_url ,
mailbox_email = mailbox . email ,
) ,
max_nb_alert = 10 ,
ignore_smtp_error = True ,
)
else :
2021-01-11 14:55:55 +01:00
LOG . d (
" Inform user %s about a bounce from contact %s to alias %s " ,
user ,
contact ,
alias ,
)
disable_alias_link = f " { URL } /dashboard/unsubscribe/ { alias . id } "
2021-10-28 18:41:36 +02:00
block_sender_link = f " { URL } /dashboard/alias_contact_manager/ { alias . id } ?highlight_contact_id= { contact . id } "
2022-01-24 16:10:36 +01:00
Notification . create (
user_id = user . id ,
title = f " Email from { contact . website_email } to { alias . email } cannot be delivered to { mailbox . email } " ,
message = Notification . render (
" notification/bounce-forward-phase.html " ,
alias = alias ,
website_email = contact . website_email ,
disable_alias_link = disable_alias_link ,
refused_email_url = refused_email . get_url ( ) ,
mailbox_email = mailbox . email ,
block_sender_link = block_sender_link ,
) ,
commit = True ,
)
2021-01-11 14:55:55 +01:00
send_email_with_rate_control (
user ,
ALERT_BOUNCE_EMAIL ,
user . email ,
f " Email from { contact . website_email } to { alias . email } cannot be delivered to your mailbox " ,
render (
2021-10-28 11:43:44 +02:00
" transactional/bounce/bounced-email.txt.jinja2 " ,
2021-01-11 14:55:55 +01:00
alias = alias ,
website_email = contact . website_email ,
disable_alias_link = disable_alias_link ,
2021-10-28 11:43:44 +02:00
block_sender_link = block_sender_link ,
2021-01-11 14:55:55 +01:00
refused_email_url = refused_email_url ,
mailbox_email = mailbox . email ,
) ,
render (
" transactional/bounce/bounced-email.html " ,
alias = alias ,
website_email = contact . website_email ,
disable_alias_link = disable_alias_link ,
refused_email_url = refused_email_url ,
mailbox_email = mailbox . email ,
) ,
max_nb_alert = 10 ,
2021-09-20 13:51:16 +02:00
# smtp error can happen if user mailbox is unreachable, that might explain the bounce
ignore_smtp_error = True ,
2021-01-11 14:55:55 +01:00
)
2020-11-04 15:38:26 +01:00
2021-09-09 18:54:54 +02:00
def handle_hotmail_complaint ( msg : Message ) - > bool :
2021-09-06 19:44:18 +02:00
"""
Handle hotmail complaint sent to postmaster
2021-09-09 18:54:54 +02:00
Return True if the complaint can be handled , False otherwise
2021-09-06 19:44:18 +02:00
"""
2021-09-13 19:50:15 +02:00
orig_msg = get_orig_message_from_hotmail_complaint ( msg )
2021-10-11 12:10:18 +02:00
to_header = orig_msg [ headers . TO ]
2021-10-26 12:16:57 +02:00
from_header = orig_msg [ headers . FROM ]
2021-11-17 14:32:30 +01:00
user = User . get_by ( email = to_header )
if user :
LOG . d ( " Handle transactional hotmail complaint for %s " , user )
handle_hotmail_complain_for_transactional_email ( user )
return True
2021-11-21 11:31:28 +01:00
try :
_ , from_address = parse_full_address ( get_header_unicode ( from_header ) )
alias = Alias . get_by ( email = from_address )
# the email is during a reply phase, from=alias and to=destination
if alias :
user = alias . user
LOG . i (
" Hotmail complaint during reply phase %s -> %s , %s " ,
alias ,
to_header ,
user ,
)
send_email_with_rate_control (
user ,
ALERT_HOTMAIL_COMPLAINT_REPLY_PHASE ,
user . email ,
f " Hotmail abuse report " ,
render (
" transactional/hotmail-complaint-reply-phase.txt.jinja2 " ,
user = user ,
alias = alias ,
destination = to_header ,
) ,
max_nb_alert = 1 ,
nb_day = 7 ,
)
return True
except ValueError :
LOG . w ( " Cannot parse %s " , from_header )
2021-11-04 10:40:12 +01:00
alias = None
2021-10-26 12:16:57 +02:00
2022-01-16 11:47:50 +01:00
# try parsing the from_header which might contain the reverse alias
2021-10-25 10:36:50 +02:00
try :
2021-10-26 12:16:57 +02:00
_ , reverse_alias = parse_full_address ( get_header_unicode ( from_header ) )
contact = Contact . get_by ( reply_email = reverse_alias )
if contact :
alias = contact . alias
LOG . d ( " find %s through %s " , alias , contact )
else :
2021-11-04 10:40:12 +01:00
LOG . d ( " No contact found for %s " , reverse_alias )
except ValueError :
LOG . w ( " Cannot parse %s " , from_header )
2021-09-06 19:44:18 +02:00
2021-11-04 10:40:12 +01:00
# try parsing the to_header which is usually the alias
2021-09-06 19:44:18 +02:00
if not alias :
2021-11-04 10:40:12 +01:00
try :
_ , alias_address = parse_full_address ( get_header_unicode ( to_header ) )
except ValueError :
LOG . w ( " Cannot parse %s " , to_header )
2022-01-16 11:47:50 +01:00
else :
alias = Alias . get_by ( email = alias_address )
if not alias :
if DeletedAlias . get_by (
email = alias_address
) or DomainDeletedAlias . get_by ( email = alias_address ) :
LOG . w ( " Alias %s is deleted " , alias_address )
return True
2021-11-04 10:40:12 +01:00
if not alias :
LOG . e (
2021-10-26 12:16:57 +02:00
" Cannot parse alias from to header %s and from header %s " ,
to_header ,
from_header ,
)
2021-09-09 18:54:54 +02:00
return False
2021-09-06 19:44:18 +02:00
user = alias . user
2021-11-04 10:40:12 +01:00
LOG . d ( " Handle hotmail complaint for %s %s %s " , alias , user , alias . mailboxes )
2021-09-06 19:44:18 +02:00
send_email_with_rate_control (
user ,
ALERT_HOTMAIL_COMPLAINT ,
user . email ,
f " Hotmail abuse report " ,
render (
" transactional/hotmail-complaint.txt.jinja2 " ,
alias = alias ,
) ,
render (
" transactional/hotmail-complaint.html " ,
alias = alias ,
) ,
2021-11-10 10:57:22 +01:00
max_nb_alert = 1 ,
nb_day = 7 ,
2021-09-06 19:44:18 +02:00
)
2021-09-09 18:54:54 +02:00
return True
2021-09-06 19:44:18 +02:00
2021-11-17 14:32:30 +01:00
def handle_hotmail_complain_for_transactional_email ( user ) :
""" Handle the case when a transactional email is set as Spam by user or by HotMail """
send_email_with_rate_control (
user ,
2021-11-21 11:31:28 +01:00
ALERT_HOTMAIL_COMPLAINT_TRANSACTIONAL ,
2021-11-17 14:32:30 +01:00
user . email ,
f " Hotmail abuse report " ,
render ( " transactional/hotmail-transactional-complaint.txt.jinja2 " , user = user ) ,
render ( " transactional/hotmail-transactional-complaint.html " , user = user ) ,
max_nb_alert = 1 ,
nb_day = 7 ,
)
return True
2021-09-13 19:49:40 +02:00
def handle_yahoo_complaint ( msg : Message ) - > bool :
"""
Handle yahoo complaint sent to postmaster
Return True if the complaint can be handled , False otherwise
"""
orig_msg = get_orig_message_from_yahoo_complaint ( msg )
2021-10-11 12:10:18 +02:00
to_header = orig_msg [ headers . TO ]
2021-09-13 19:49:40 +02:00
if not to_header :
LOG . e ( " cannot find the alias " )
return False
2021-11-17 14:37:35 +01:00
user = User . get_by ( email = to_header )
if user :
LOG . d ( " Handle transactional yahoo complaint for %s " , user )
handle_yahoo_complain_for_transactional_email ( user )
return True
2021-09-13 19:49:40 +02:00
_ , alias_address = parse_full_address ( get_header_unicode ( to_header ) )
alias = Alias . get_by ( email = alias_address )
if not alias :
LOG . w ( " No alias for %s " , alias_address )
return False
user = alias . user
LOG . w ( " Handle yahoo complaint for %s %s %s " , alias , user , alias . mailboxes )
send_email_with_rate_control (
user ,
ALERT_YAHOO_COMPLAINT ,
user . email ,
f " Yahoo abuse report " ,
render (
" transactional/yahoo-complaint.txt.jinja2 " ,
alias = alias ,
) ,
render (
" transactional/yahoo-complaint.html " ,
alias = alias ,
) ,
max_nb_alert = 2 ,
)
return True
2021-11-17 14:37:35 +01:00
def handle_yahoo_complain_for_transactional_email ( user ) :
""" Handle the case when a transactional email is set as Spam by user or by Yahoo """
send_email_with_rate_control (
user ,
ALERT_YAHOO_COMPLAINT ,
user . email ,
f " Yahoo abuse report " ,
render ( " transactional/yahoo-transactional-complaint.txt.jinja2 " , user = user ) ,
render ( " transactional/yahoo-transactional-complaint.html " , user = user ) ,
max_nb_alert = 1 ,
nb_day = 7 ,
)
return True
2021-03-06 17:44:31 +01:00
def handle_bounce_reply_phase ( envelope , msg : Message , email_log : EmailLog ) :
2021-01-11 14:55:55 +01:00
"""
Handle reply phase bounce
Happens when an email cannot be sent from an alias to a contact
"""
2021-01-26 09:56:13 +01:00
contact : Contact = email_log . contact
2021-01-11 14:55:55 +01:00
alias = contact . alias
user = alias . user
2021-01-11 15:53:08 +01:00
mailbox = email_log . mailbox or alias . mailbox
2021-09-08 11:29:55 +02:00
LOG . d ( " Handle reply bounce %s -> %s -> %s . %s " , mailbox , alias , contact , email_log )
2021-01-11 14:55:55 +01:00
2021-10-14 15:46:52 +02:00
bounce_info = get_mailbox_bounce_info ( msg )
if bounce_info :
Bounce . create (
2021-11-18 16:44:04 +01:00
email = sanitize_email ( contact . website_email , not_lower = True ) ,
2021-10-19 12:05:35 +02:00
info = bounce_info . as_bytes ( ) . decode ( ) ,
2021-10-14 15:46:52 +02:00
commit = True ,
)
else :
2021-11-02 14:32:16 +01:00
LOG . w ( " cannot get bounce info, debug at %s " , save_email_for_debugging ( msg ) )
2021-11-18 16:44:04 +01:00
Bounce . create (
email = sanitize_email ( contact . website_email , not_lower = True ) , commit = True
)
2021-01-26 09:56:13 +01:00
2020-11-04 12:32:15 +01:00
# Store the bounced email
# generate a name for the email
random_name = str ( uuid . uuid4 ( ) )
full_report_path = f " refused-emails/full- { random_name } .eml "
2020-11-09 17:02:10 +01:00
s3 . upload_email_from_bytesio ( full_report_path , BytesIO ( to_bytes ( msg ) ) , random_name )
2020-11-04 12:32:15 +01:00
orig_msg = get_orig_message_from_bounce ( msg )
2020-11-04 15:38:26 +01:00
file_path = None
if orig_msg :
file_path = f " refused-emails/ { random_name } .eml "
s3 . upload_email_from_bytesio (
2020-11-09 17:02:10 +01:00
file_path , BytesIO ( to_bytes ( orig_msg ) ) , random_name
2020-11-04 15:38:26 +01:00
)
2020-11-04 12:32:15 +01:00
2020-11-04 15:38:26 +01:00
refused_email = RefusedEmail . create (
path = file_path , full_report_path = full_report_path , user_id = user . id , commit = True
)
LOG . d ( " Create refused email %s " , refused_email )
2020-11-04 12:32:15 +01:00
2020-11-04 15:38:26 +01:00
email_log . bounced = True
email_log . refused_email_id = refused_email . id
2020-11-04 12:32:15 +01:00
2020-11-24 16:50:55 +01:00
email_log . bounced_mailbox_id = mailbox . id
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-11-04 12:32:15 +01:00
2020-12-06 13:54:59 +01:00
refused_email_url = f " { URL } /dashboard/refused_email?highlight_id= { email_log . id } "
2020-11-04 12:32:15 +01:00
LOG . d (
" Inform user %s about bounced email sent by %s to %s " ,
user ,
alias ,
contact ,
)
2022-01-24 16:13:45 +01:00
Notification . create (
user_id = user . id ,
title = f " Email cannot be sent to { contact . email } from your alias { alias . email } " ,
message = Notification . render (
" notification/bounce-reply-phase.html " ,
alias = alias ,
contact = contact ,
refused_email_url = refused_email . get_url ( ) ,
) ,
commit = True ,
)
2020-11-04 12:32:15 +01:00
send_email_with_rate_control (
user ,
ALERT_BOUNCE_EMAIL_REPLY_PHASE ,
mailbox . email ,
f " Email cannot be sent to { contact . email } from your alias { alias . email } " ,
render (
2021-01-04 14:22:07 +01:00
" transactional/bounce/bounce-email-reply-phase.txt " ,
2020-11-04 12:32:15 +01:00
alias = alias ,
contact = contact ,
refused_email_url = refused_email_url ,
) ,
render (
2021-01-04 14:22:07 +01:00
" transactional/bounce/bounce-email-reply-phase.html " ,
2020-11-04 12:32:15 +01:00
alias = alias ,
contact = contact ,
refused_email_url = refused_email_url ,
) ,
)
2020-03-30 22:05:31 +02:00
def handle_spam (
contact : Contact ,
alias : Alias ,
msg : Message ,
user : User ,
2020-08-21 10:18:58 +02:00
mailbox : Mailbox ,
2020-04-27 18:18:40 +02:00
email_log : EmailLog ,
2020-08-15 16:53:57 +02:00
is_reply = False , # whether the email is in forward or reply phase
2020-03-30 22:05:31 +02:00
) :
# Store the report & original email
orig_msg = get_orig_message_from_spamassassin_report ( msg )
# generate a name for the email
random_name = str ( uuid . uuid4 ( ) )
2020-04-02 18:09:05 +02:00
full_report_path = f " spams/full- { random_name } .eml "
2020-11-09 17:02:10 +01:00
s3 . upload_email_from_bytesio ( full_report_path , BytesIO ( to_bytes ( msg ) ) , random_name )
2020-03-30 22:05:31 +02:00
file_path = None
if orig_msg :
2020-04-02 18:09:05 +02:00
file_path = f " spams/ { random_name } .eml "
2020-03-30 22:05:31 +02:00
s3 . upload_email_from_bytesio (
2020-11-09 17:02:10 +01:00
file_path , BytesIO ( to_bytes ( orig_msg ) ) , random_name
2020-03-30 22:05:31 +02:00
)
refused_email = RefusedEmail . create (
path = file_path , full_report_path = full_report_path , user_id = user . id
)
2021-10-12 14:36:47 +02:00
Session . flush ( )
2020-03-30 22:05:31 +02:00
email_log . refused_email_id = refused_email . id
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-03-30 22:05:31 +02:00
LOG . d ( " Create spam email %s " , refused_email )
2020-12-06 13:54:59 +01:00
refused_email_url = f " { URL } /dashboard/refused_email?highlight_id= { email_log . id } "
2020-03-30 22:05:31 +02:00
disable_alias_link = f " { URL } /dashboard/unsubscribe/ { alias . id } "
2020-08-15 16:53:57 +02:00
if is_reply :
LOG . d (
2020-12-02 12:40:29 +01:00
" Inform %s ( %s ) about spam email sent from alias %s to %s . %s " ,
2020-08-21 10:20:08 +02:00
mailbox ,
2020-08-15 16:53:57 +02:00
user ,
alias ,
contact ,
2020-12-02 12:40:29 +01:00
refused_email ,
2020-08-15 16:53:57 +02:00
)
send_email_with_rate_control (
user ,
ALERT_SPAM_EMAIL ,
2020-08-21 10:18:58 +02:00
mailbox . email ,
2020-11-18 16:16:37 +01:00
f " Email from { alias . email } to { contact . website_email } is detected as spam " ,
2020-08-15 16:53:57 +02:00
render (
" transactional/spam-email-reply-phase.txt " ,
alias = alias ,
website_email = contact . website_email ,
disable_alias_link = disable_alias_link ,
refused_email_url = refused_email_url ,
) ,
render (
" transactional/spam-email-reply-phase.html " ,
alias = alias ,
website_email = contact . website_email ,
disable_alias_link = disable_alias_link ,
refused_email_url = refused_email_url ,
) ,
)
else :
# inform user
LOG . d (
2020-08-21 10:20:08 +02:00
" Inform %s ( %s ) about spam email sent by %s to alias %s " ,
mailbox ,
2020-08-15 16:53:57 +02:00
user ,
contact ,
alias ,
)
send_email_with_rate_control (
user ,
ALERT_SPAM_EMAIL ,
2020-08-21 10:18:58 +02:00
mailbox . email ,
2020-08-15 16:53:57 +02:00
f " Email from { contact . website_email } to { alias . email } is detected as spam " ,
render (
" transactional/spam-email.txt " ,
alias = alias ,
website_email = contact . website_email ,
disable_alias_link = disable_alias_link ,
refused_email_url = refused_email_url ,
) ,
render (
" transactional/spam-email.html " ,
alias = alias ,
website_email = contact . website_email ,
disable_alias_link = disable_alias_link ,
refused_email_url = refused_email_url ,
) ,
)
2020-03-30 22:05:31 +02:00
2022-01-04 18:06:08 +01:00
def is_automatic_out_of_office ( msg : Message ) - > bool :
2022-01-05 15:21:54 +01:00
"""
Return whether an email is out - of - office
For info , out - of - office is sent to the envelope mail_from and not the From : header
More info on https : / / datatracker . ietf . org / doc / html / rfc3834 #section-4 and https://support.google.com/mail/thread/21246740/my-auto-reply-filter-isn-t-replying-to-original-sender-address?hl=en&msgid=21261237
"""
if msg [ headers . AUTO_SUBMITTED ] is None :
return False
2022-01-04 18:06:08 +01:00
2022-01-05 15:21:54 +01:00
if msg [ headers . AUTO_SUBMITTED ] . lower ( ) in ( " auto-replied " , " auto-generated " ) :
2022-01-04 18:06:08 +01:00
LOG . d (
2022-01-05 15:21:54 +01:00
" out-of-office email %s : %s " ,
headers . AUTO_SUBMITTED ,
msg [ headers . AUTO_SUBMITTED ] ,
2022-01-04 18:06:08 +01:00
)
return True
return False
2022-01-05 15:22:22 +01:00
def is_bounce ( envelope : Envelope , msg : Message ) :
""" Detect whether an email is a Delivery Status Notification """
return (
envelope . mail_from == " <> "
and msg . get_content_type ( ) . lower ( ) == " multipart/report "
)
2021-03-17 10:20:10 +01:00
def handle_unsubscribe ( envelope : Envelope , msg : Message ) - > str :
2020-10-22 10:34:52 +02:00
""" return the SMTP status """
2020-03-28 23:19:25 +01:00
# format: alias_id:
2021-10-11 12:10:18 +02:00
subject = msg [ headers . SUBJECT ]
2021-11-02 15:44:43 +01:00
alias , contact = None , None
2020-03-28 23:19:25 +01:00
try :
2020-04-24 09:09:11 +02:00
# subject has the format {alias.id}=
if subject . endswith ( " = " ) :
alias_id = int ( subject [ : - 1 ] )
2021-11-02 15:44:43 +01:00
alias = Alias . get ( alias_id )
# {contact.id}_
elif subject . endswith ( " _ " ) :
contact_id = int ( subject [ : - 1 ] )
contact = Contact . get ( contact_id )
if contact :
alias = contact . alias
2020-10-22 10:34:52 +02:00
# {user.id}*
elif subject . endswith ( " * " ) :
user_id = int ( subject [ : - 1 ] )
return handle_unsubscribe_user ( user_id , envelope . mail_from )
2020-04-24 09:09:11 +02:00
# some email providers might strip off the = suffix
else :
alias_id = int ( subject )
2021-11-02 15:44:43 +01:00
alias = Alias . get ( alias_id )
2020-03-28 23:19:25 +01:00
except Exception :
2022-03-08 10:31:20 +01:00
LOG . w ( " Wrong format subject %s " , msg [ headers . SUBJECT ] )
2021-06-23 19:47:06 +02:00
return status . E507
2020-03-28 23:19:25 +01:00
if not alias :
2022-03-08 10:31:20 +01:00
LOG . w ( " Cannot get alias from subject %s " , subject )
2021-06-23 19:47:06 +02:00
return status . E508
2020-03-28 23:19:25 +01:00
2020-09-14 17:38:48 +02:00
mail_from = envelope . mail_from
2020-10-23 13:29:20 +02:00
# Only alias's owning mailbox can send the unsubscribe request
mailbox = get_mailbox_from_mail_from ( mail_from , alias )
if not mailbox :
2022-03-08 18:35:18 +01:00
LOG . d (
" %s cannot disable alias %s . Alias authorized addresses: %s " ,
envelope . mail_from ,
alias ,
alias . authorized_addresses ,
)
2021-06-23 19:47:06 +02:00
return status . E509
2020-03-28 23:19:25 +01:00
user = alias . user
2021-11-02 15:44:43 +01:00
if contact :
contact . block_forward = True
Session . commit ( )
unblock_contact_url = (
URL
2021-11-03 10:11:52 +01:00
+ f " /dashboard/alias_contact_manager/ { alias . id } ?highlight_contact_id= { contact . id } "
2020-05-10 18:23:43 +02:00
)
2021-11-02 15:44:43 +01:00
for mailbox in alias . mailboxes :
send_email (
mailbox . email ,
f " Emails from { contact . website_email } to { alias . email } are now blocked " ,
render (
" transactional/unsubscribe-block-contact.txt.jinja2 " ,
user = user ,
alias = alias ,
contact = contact ,
unblock_contact_url = unblock_contact_url ,
) ,
)
else :
alias . enabled = False
Session . commit ( )
enable_alias_url = URL + f " /dashboard/?highlight_alias_id= { alias . id } "
for mailbox in alias . mailboxes :
send_email (
mailbox . email ,
f " Alias { alias . email } has been disabled successfully " ,
render (
" transactional/unsubscribe-disable-alias.txt " ,
user = user ,
alias = alias . email ,
enable_alias_url = enable_alias_url ,
) ,
render (
" transactional/unsubscribe-disable-alias.html " ,
user = user ,
alias = alias . email ,
enable_alias_url = enable_alias_url ,
) ,
)
2020-03-28 23:19:25 +01:00
2021-06-23 19:47:06 +02:00
return status . E202
2020-03-28 23:19:25 +01:00
2020-10-22 10:34:52 +02:00
def handle_unsubscribe_user ( user_id : int , mail_from : str ) - > str :
""" return the SMTP status """
user = User . get ( user_id )
if not user :
2021-09-20 16:59:27 +02:00
LOG . w ( " No such user %s %s " , user_id , mail_from )
2021-06-23 19:47:06 +02:00
return status . E510
2020-10-22 10:34:52 +02:00
if mail_from != user . email :
2021-09-08 11:29:55 +02:00
LOG . e ( " Unauthorized mail_from %s %s " , user , mail_from )
2021-06-23 19:47:06 +02:00
return status . E511
2020-10-22 10:34:52 +02:00
user . notification = False
2021-10-12 14:36:47 +02:00
Session . commit ( )
2020-10-22 10:34:52 +02:00
send_email (
user . email ,
2020-12-06 13:54:59 +01:00
" You have been unsubscribed from SimpleLogin newsletter " ,
2020-10-22 10:34:52 +02:00
render (
" transactional/unsubscribe-newsletter.txt " ,
user = user ,
) ,
render (
" transactional/unsubscribe-newsletter.html " ,
user = user ,
) ,
)
2021-06-23 19:47:06 +02:00
return status . E202
2020-10-22 12:26:45 +02:00
2020-10-22 10:34:52 +02:00
2021-10-14 15:46:52 +02:00
def handle_transactional_bounce ( envelope : Envelope , msg , rcpt_to ) :
2021-01-26 09:59:08 +01:00
LOG . d ( " handle transactional bounce sent to %s " , rcpt_to )
2020-06-10 13:57:23 +02:00
2021-01-26 09:59:08 +01:00
# parse the TransactionalEmail
transactional_id = parse_id_from_bounce ( rcpt_to )
transactional = TransactionalEmail . get ( transactional_id )
2020-06-10 13:57:23 +02:00
2021-02-06 16:00:32 +01:00
# a transaction might have been deleted in delete_logs()
2021-01-26 09:59:08 +01:00
if transactional :
2021-03-16 09:17:23 +01:00
LOG . i ( " Create bounce for %s " , transactional . email )
2021-10-14 15:46:52 +02:00
bounce_info = get_mailbox_bounce_info ( msg )
if bounce_info :
Bounce . create (
2021-10-19 12:05:35 +02:00
email = transactional . email ,
info = bounce_info . as_bytes ( ) . decode ( ) ,
commit = True ,
2021-10-14 15:46:52 +02:00
)
else :
2021-11-02 14:32:16 +01:00
LOG . w ( " cannot get bounce info, debug at %s " , save_email_for_debugging ( msg ) )
2021-10-14 15:46:52 +02:00
Bounce . create ( email = transactional . email , commit = True )
2020-06-10 13:57:23 +02:00
2021-06-02 11:38:25 +02:00
def handle_bounce ( envelope , email_log : EmailLog , msg : Message ) - > str :
2021-03-17 10:09:54 +01:00
"""
Return SMTP status , e . g . " 500 Error "
"""
if not email_log :
2021-06-02 11:38:25 +02:00
LOG . w ( " No such email log " )
2021-06-23 19:47:06 +02:00
return status . E512
2021-03-17 10:09:54 +01:00
2021-06-02 11:38:25 +02:00
contact : Contact = email_log . contact
alias = contact . alias
LOG . d (
" handle bounce for %s , phase= %s , contact= %s , alias= %s " ,
email_log ,
email_log . get_phase ( ) ,
contact ,
alias ,
)
2021-03-17 10:09:54 +01:00
if email_log . is_reply :
content_type = msg . get_content_type ( ) . lower ( )
if content_type != " multipart/report " or envelope . mail_from != " <> " :
# forward the email again to the alias
2021-05-30 20:02:41 +02:00
LOG . i (
" Handle auto reply %s %s " ,
2021-03-17 10:09:54 +01:00
content_type ,
envelope . mail_from ,
)
contact : Contact = email_log . contact
alias = contact . alias
email_log . auto_replied = True
2021-10-12 14:36:47 +02:00
Session . commit ( )
2021-03-17 10:09:54 +01:00
# replace the BOUNCE_EMAIL by alias in To field
add_or_replace_header ( msg , " To " , alias . email )
envelope . rcpt_tos = [ alias . email ]
# same as handle()
# result of all deliveries
# each element is a couple of whether the delivery is successful and the smtp status
res : [ ( bool , str ) ] = [ ]
for is_delivered , smtp_status in handle_forward ( envelope , msg , alias . email ) :
res . append ( ( is_delivered , smtp_status ) )
for ( is_success , smtp_status ) in res :
# Consider all deliveries successful if 1 delivery is successful
if is_success :
return smtp_status
# Failed delivery for all, return the first failure
return res [ 0 ] [ 1 ]
2021-11-13 11:21:19 +01:00
handle_bounce_reply_phase ( envelope , msg , email_log )
return status . E212
2021-03-17 10:09:54 +01:00
else : # forward phase
handle_bounce_forward_phase ( msg , email_log )
2021-11-13 11:21:19 +01:00
return status . E211
2021-03-17 10:09:54 +01:00
2021-06-22 17:52:24 +02:00
def should_ignore ( mail_from : str , rcpt_tos : List [ str ] ) - > bool :
if len ( rcpt_tos ) != 1 :
return False
rcpt_to = rcpt_tos [ 0 ]
if IgnoredEmail . get_by ( mail_from = mail_from , rcpt_to = rcpt_to ) :
return True
return False
2022-02-16 18:38:31 +01:00
def send_no_reply_response ( mail_from : str , msg : Message ) :
2022-02-17 14:33:04 +01:00
mailbox = Mailbox . get_by ( email = mail_from )
2022-02-21 12:30:26 +01:00
if not mailbox :
2022-02-16 18:39:18 +01:00
LOG . d ( " Unknown sender. Skipping reply from {} " . format ( NOREPLY ) )
2022-02-17 14:33:04 +01:00
return
send_email_at_most_times (
2022-02-21 12:30:26 +01:00
mailbox . user ,
2022-02-17 14:33:04 +01:00
ALERT_TO_NOREPLY ,
2022-02-21 12:30:26 +01:00
mailbox . user . email ,
2022-02-17 14:33:04 +01:00
" Auto: {} " . format ( msg [ headers . SUBJECT ] or " No subject " ) ,
render ( " transactional/noreply.text.jinja2 " ) ,
)
2022-02-16 18:38:31 +01:00
2022-01-07 14:26:58 +01:00
def handle ( envelope : Envelope , msg : Message ) - > str :
2020-04-04 16:09:24 +02:00
""" Return SMTP status """
2020-09-14 17:19:29 +02:00
# sanitize mail_from, rcpt_tos
2021-01-11 12:27:02 +01:00
mail_from = sanitize_email ( envelope . mail_from )
rcpt_tos = [ sanitize_email ( rcpt_to ) for rcpt_to in envelope . rcpt_tos ]
2020-09-14 17:19:29 +02:00
envelope . mail_from = mail_from
envelope . rcpt_tos = rcpt_tos
2021-06-04 17:15:59 +02:00
postfix_queue_id = get_queue_id ( msg )
if postfix_queue_id :
set_message_id ( postfix_queue_id )
2021-06-23 18:19:13 +02:00
else :
2021-10-15 10:37:22 +02:00
LOG . d (
" Cannot parse Postfix queue ID from %s %s " ,
msg . get_all ( headers . RECEIVED ) ,
msg [ headers . RECEIVED ] ,
)
2021-06-23 18:19:13 +02:00
if should_ignore ( mail_from , rcpt_tos ) :
2021-06-23 19:47:51 +02:00
LOG . w ( " Ignore email mail_from= %s rcpt_to= %s " , mail_from , rcpt_tos )
2021-06-23 19:47:06 +02:00
return status . E204
2021-03-17 10:19:27 +01:00
# sanitize email headers
2021-03-17 10:59:13 +01:00
sanitize_header ( msg , " from " )
sanitize_header ( msg , " to " )
sanitize_header ( msg , " cc " )
sanitize_header ( msg , " reply-to " )
2021-03-17 10:19:27 +01:00
2021-03-16 09:17:23 +01:00
LOG . d (
" ==>> Handle mail_from: %s , rcpt_tos: %s , header_from: %s , header_to: %s , "
2022-03-16 10:25:28 +01:00
" cc: %s , reply-to: %s , message_id: %s , client_ip: %s , headers: %s , mail_options: %s , rcpt_options: %s " ,
2021-03-16 09:17:23 +01:00
mail_from ,
rcpt_tos ,
2021-10-11 12:10:18 +02:00
msg [ headers . FROM ] ,
msg [ headers . TO ] ,
msg [ headers . CC ] ,
msg [ headers . REPLY_TO ] ,
2021-10-19 12:05:41 +02:00
msg [ headers . MESSAGE_ID ] ,
2022-03-16 10:25:28 +01:00
msg [ headers . SL_CLIENT_IP ] ,
msg . _headers ,
2021-03-16 09:17:23 +01:00
envelope . mail_options ,
envelope . rcpt_options ,
)
2022-01-07 16:14:21 +01:00
# region mail_from or from_header is a reverse alias which should never happen
2022-01-08 00:42:03 +01:00
email_sent_from_reverse_alias = False
2021-03-15 19:55:22 +01:00
contact = Contact . get_by ( reply_email = mail_from )
if contact :
2022-01-08 00:42:03 +01:00
email_sent_from_reverse_alias = True
2022-01-07 16:14:21 +01:00
from_header = get_header_unicode ( msg [ headers . FROM ] )
if from_header :
try :
_ , from_header_address = parse_full_address ( from_header )
except ValueError :
2022-01-08 00:43:49 +01:00
LOG . w ( " cannot parse the From header %s " , from_header )
2022-01-07 16:14:21 +01:00
else :
contact = Contact . get_by ( reply_email = from_header_address )
if contact :
2022-01-08 00:42:03 +01:00
email_sent_from_reverse_alias = True
if email_sent_from_reverse_alias :
LOG . w ( f " email sent from reverse alias { contact } { contact . alias } { contact . user } " )
user = contact . user
send_email_at_most_times (
user ,
ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS ,
user . email ,
" SimpleLogin shouldn ' t be used with another email forwarding system " ,
render (
" transactional/email-sent-from-reverse-alias.txt.jinja2 " ,
) ,
)
2022-01-07 16:14:21 +01:00
# endregion
2021-03-15 19:55:22 +01:00
2020-04-04 16:09:24 +02:00
# unsubscribe request
2021-11-22 18:17:07 +01:00
if UNSUBSCRIBER and ( rcpt_tos == [ UNSUBSCRIBER ] or rcpt_tos == [ OLD_UNSUBSCRIBER ] ) :
2020-09-14 17:19:29 +02:00
LOG . d ( " Handle unsubscribe request from %s " , mail_from )
2021-03-17 10:20:10 +01:00
return handle_unsubscribe ( envelope , msg )
2020-04-04 16:09:24 +02:00
2022-01-07 16:14:21 +01:00
# region mail sent to VERP
2022-01-07 12:18:46 +01:00
# sent to transactional VERP. Either bounce emails or out-of-office
2021-01-26 09:59:08 +01:00
if (
len ( rcpt_tos ) == 1
and rcpt_tos [ 0 ] . startswith ( TRANSACTIONAL_BOUNCE_PREFIX )
and rcpt_tos [ 0 ] . endswith ( TRANSACTIONAL_BOUNCE_SUFFIX )
) :
2022-01-05 15:30:44 +01:00
if is_bounce ( envelope , msg ) :
handle_transactional_bounce ( envelope , msg , rcpt_tos [ 0 ] )
return status . E205
elif is_automatic_out_of_office ( msg ) :
LOG . d (
" Ignore out-of-office for transactional emails. Headers: %s " , msg . items
)
return status . E206
else :
2022-01-07 14:57:47 +01:00
raise VERPTransactional
2020-06-10 13:57:23 +02:00
2022-01-07 12:18:46 +01:00
# sent to forward VERP, can be either bounce or out-of-office
2021-01-11 14:55:55 +01:00
if (
len ( rcpt_tos ) == 1
and rcpt_tos [ 0 ] . startswith ( BOUNCE_PREFIX )
and rcpt_tos [ 0 ] . endswith ( BOUNCE_SUFFIX )
) :
2021-06-02 11:38:25 +02:00
email_log_id = parse_id_from_bounce ( rcpt_tos [ 0 ] )
email_log = EmailLog . get ( email_log_id )
2021-05-25 17:59:09 +02:00
2022-01-05 15:30:44 +01:00
if not email_log :
LOG . w ( " No such email log " )
return status . E512
if is_bounce ( envelope , msg ) :
2022-01-05 09:50:58 +01:00
return handle_bounce ( envelope , email_log , msg )
2022-01-05 15:30:44 +01:00
elif is_automatic_out_of_office ( msg ) :
2022-01-07 12:24:14 +01:00
handle_out_of_office_forward_phase ( email_log , envelope , msg , rcpt_tos )
2022-01-05 15:30:44 +01:00
else :
2022-01-07 14:57:47 +01:00
raise VERPForward
2022-01-05 09:50:58 +01:00
2022-01-07 12:18:46 +01:00
# sent to reply VERP, can be either bounce or out-of-office
2022-01-05 09:50:58 +01:00
if len ( rcpt_tos ) == 1 and rcpt_tos [ 0 ] . startswith (
f " { BOUNCE_PREFIX_FOR_REPLY_PHASE } + "
2021-05-25 17:59:09 +02:00
) :
2021-06-02 11:38:25 +02:00
email_log_id = parse_id_from_bounce ( rcpt_tos [ 0 ] )
email_log = EmailLog . get ( email_log_id )
2022-01-05 09:50:58 +01:00
2022-01-05 15:30:44 +01:00
if not email_log :
LOG . w ( " No such email log " )
return status . E512
2022-01-06 14:35:16 +01:00
# bounce by contact
2022-01-05 15:30:44 +01:00
if is_bounce ( envelope , msg ) :
return handle_bounce ( envelope , email_log , msg )
elif is_automatic_out_of_office ( msg ) :
2022-01-07 12:24:14 +01:00
handle_out_of_office_reply_phase ( email_log , envelope , msg , rcpt_tos )
2022-01-05 15:30:44 +01:00
else :
2022-01-07 14:57:47 +01:00
raise VERPReply (
f " cannot handle email sent to reply VERP, "
f " { email_log . alias } -> { email_log . contact } ( { email_log } , { email_log . user } "
2022-01-05 15:30:44 +01:00
)
2021-01-11 14:55:55 +01:00
2021-06-02 11:46:00 +02:00
# iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias
2021-06-02 16:27:48 +02:00
if (
len ( rcpt_tos ) == 1
and mail_from . startswith ( BOUNCE_PREFIX )
and mail_from . endswith ( BOUNCE_SUFFIX )
) :
2021-06-02 11:46:00 +02:00
email_log_id = parse_id_from_bounce ( mail_from )
email_log = EmailLog . get ( email_log_id )
alias = Alias . get_by ( email = rcpt_tos [ 0 ] )
2021-06-04 15:23:48 +02:00
LOG . w (
2021-06-02 11:46:00 +02:00
" iCloud bounces %s %s msg= %s " ,
email_log ,
alias ,
msg . as_string ( ) ,
)
return handle_bounce ( envelope , email_log , msg )
2022-01-07 12:18:46 +01:00
# endregion
2022-01-07 16:14:21 +01:00
# region hotmail, yahoo complaints
2022-01-07 12:18:46 +01:00
if (
len ( rcpt_tos ) == 1
and mail_from == " staff@hotmail.com "
and rcpt_tos [ 0 ] == POSTMASTER
) :
LOG . w ( " Handle hotmail complaint " )
# if the complaint cannot be handled, forward it normally
if handle_hotmail_complaint ( msg ) :
return status . E208
if (
len ( rcpt_tos ) == 1
and mail_from == " feedback@arf.mail.yahoo.com "
and rcpt_tos [ 0 ] == POSTMASTER
) :
LOG . w ( " Handle yahoo complaint " )
# if the complaint cannot be handled, forward it normally
if handle_yahoo_complaint ( msg ) :
return status . E210
2022-01-07 16:14:21 +01:00
# endregion
2021-09-22 09:58:40 +02:00
2021-06-24 09:47:01 +02:00
if rate_limited ( mail_from , rcpt_tos ) :
2021-10-13 10:27:54 +02:00
LOG . w ( " Rate Limiting applied for mail_from: %s rcpt_tos: %s " , mail_from , rcpt_tos )
# add more logging info. TODO: remove
if len ( rcpt_tos ) == 1 :
alias = Alias . get_by ( email = rcpt_tos [ 0 ] )
if alias :
LOG . w (
" total number email log on %s , %s is %s , %s " ,
alias ,
alias . user ,
EmailLog . filter ( EmailLog . alias_id == alias . id ) . count ( ) ,
EmailLog . filter ( EmailLog . user_id == alias . user_id ) . count ( ) ,
)
2021-10-12 14:53:30 +02:00
2021-10-13 10:27:54 +02:00
if should_ignore_bounce ( envelope . mail_from ) :
return status . E207
else :
return status . E522
2020-04-04 16:27:22 +02:00
2022-01-05 15:20:17 +01:00
# Handle "out-of-office" auto notice, i.e. an automatic response is sent for every forwarded email
2021-10-19 12:14:16 +02:00
if len ( rcpt_tos ) == 1 and is_reverse_alias ( rcpt_tos [ 0 ] ) and mail_from == " <> " :
2022-01-05 15:20:17 +01:00
contact = Contact . get_by ( reply_email = rcpt_tos [ 0 ] )
2021-05-30 20:02:41 +02:00
LOG . w (
2022-01-05 17:43:11 +01:00
" out-of-office email to reverse alias %s . Saved to %s " ,
2022-01-05 15:20:17 +01:00
contact ,
save_email_for_debugging ( msg ) , # todo: remove
2021-05-30 20:02:41 +02:00
)
2021-06-23 19:47:06 +02:00
return status . E206
2021-05-30 19:58:08 +02:00
2020-04-04 16:09:24 +02:00
# result of all deliveries
# each element is a couple of whether the delivery is successful and the smtp status
res : [ ( bool , str ) ] = [ ]
2021-05-06 17:08:30 +02:00
nb_rcpt_tos = len ( rcpt_tos )
for rcpt_index , rcpt_to in enumerate ( rcpt_tos ) :
2020-11-14 15:55:53 +01:00
if rcpt_to == NOREPLY :
2022-02-16 18:38:31 +01:00
LOG . i ( " email sent to {} address from {} " . format ( NOREPLY , mail_from ) )
send_no_reply_response ( mail_from , msg )
return status . E200
2020-11-14 15:55:53 +01:00
2021-05-06 17:08:30 +02:00
# create a copy of msg for each recipient except the last one
# as copy() is a slow function
if rcpt_index < nb_rcpt_tos - 1 :
LOG . d ( " copy message for rcpt %s " , rcpt_to )
copy_msg = copy ( msg )
else :
copy_msg = msg
2021-12-30 10:24:57 +01:00
# Reply case: the recipient is a reverse alias. Used to start with "reply+" or "ra+"
2021-10-19 12:14:16 +02:00
if is_reverse_alias ( rcpt_to ) :
2021-10-11 12:10:18 +02:00
LOG . d (
" Reply phase %s ( %s ) -> %s " , mail_from , copy_msg [ headers . FROM ] , rcpt_to
)
2021-05-06 17:08:30 +02:00
is_delivered , smtp_status = handle_reply ( envelope , copy_msg , rcpt_to )
2020-04-04 16:09:24 +02:00
res . append ( ( is_delivered , smtp_status ) )
else : # Forward case
2021-03-16 09:17:23 +01:00
LOG . d (
2020-11-04 19:42:20 +01:00
" Forward phase %s ( %s ) -> %s " ,
2020-09-14 17:19:29 +02:00
mail_from ,
2021-10-11 12:10:18 +02:00
copy_msg [ headers . FROM ] ,
2020-07-23 10:32:10 +02:00
rcpt_to ,
)
2021-05-06 17:08:30 +02:00
for is_delivered , smtp_status in handle_forward (
envelope , copy_msg , rcpt_to
) :
2020-05-10 16:57:47 +02:00
res . append ( ( is_delivered , smtp_status ) )
2020-04-04 16:09:24 +02:00
2022-03-11 08:56:27 +01:00
# to know whether both successful and unsuccessful deliveries can happen at the same time
nb_success = len ( [ is_success for ( is_success , smtp_status ) in res if is_success ] )
nb_non_success = len (
[ is_success for ( is_success , smtp_status ) in res if not is_success ]
)
2022-03-16 09:05:57 +01:00
if nb_success > 0 and nb_non_success > 0 :
LOG . e ( f " some deliveries fail and some success, { mail_from } , { rcpt_tos } , { res } " )
2022-03-11 08:56:27 +01:00
2020-04-04 16:09:24 +02:00
for ( is_success , smtp_status ) in res :
# Consider all deliveries successful if 1 delivery is successful
if is_success :
return smtp_status
# Failed delivery for all, return the first failure
return res [ 0 ] [ 1 ]
2022-01-07 12:24:14 +01:00
def handle_out_of_office_reply_phase ( email_log , envelope , msg , rcpt_tos ) :
""" convert the email into a normal email sent to the alias, so it can be forwarded to mailbox """
LOG . d (
" send the out-of-office email to the alias %s , old to_header: %s rcpt_tos: %s , %s " ,
email_log . alias ,
msg [ headers . TO ] ,
rcpt_tos ,
email_log ,
)
alias_address = email_log . alias . email
rcpt_tos [ 0 ] = alias_address
envelope . rcpt_tos = [ alias_address ]
add_or_replace_header ( msg , headers . TO , alias_address )
# delete reply-to header that can affect email delivery
delete_header ( msg , headers . REPLY_TO )
LOG . d (
" after out-of-office transformation to_header: %s reply_to: %s rcpt_tos: %s " ,
msg . get_all ( headers . TO ) ,
msg . get_all ( headers . REPLY_TO ) ,
rcpt_tos ,
)
def handle_out_of_office_forward_phase ( email_log , envelope , msg , rcpt_tos ) :
""" convert the email into a normal email sent to the reverse alias, so it can be forwarded to contact """
LOG . d (
" send the out-of-office email to the contact %s , old to_header: %s rcpt_tos: %s %s " ,
email_log . contact ,
msg [ headers . TO ] ,
rcpt_tos ,
email_log ,
)
reverse_alias = email_log . contact . reply_email
rcpt_tos [ 0 ] = reverse_alias
envelope . rcpt_tos = [ reverse_alias ]
add_or_replace_header ( msg , headers . TO , reverse_alias )
# delete reply-to header that can affect email delivery
delete_header ( msg , headers . REPLY_TO )
LOG . d (
" after out-of-office transformation to_header: %s reply_to: %s rcpt_tos: %s " ,
msg . get_all ( headers . TO ) ,
msg . get_all ( headers . REPLY_TO ) ,
rcpt_tos ,
)
2020-02-19 16:17:13 +01:00
class MailHandler :
2020-04-02 18:10:08 +02:00
async def handle_DATA ( self , server , session , envelope : Envelope ) :
2022-01-07 14:57:47 +01:00
msg = email . message_from_bytes ( envelope . original_content )
2020-08-17 11:40:58 +02:00
try :
2022-01-07 14:26:58 +01:00
ret = self . _handle ( envelope , msg )
2020-08-17 11:40:58 +02:00
return ret
2022-01-08 00:16:16 +01:00
# happen if reverse-alias is used during the forward phase
# as in this case, a new reverse-alias needs to be created for this reverse-alias -> chaos
except CannotCreateContactForReverseAlias as e :
LOG . w (
" Probably due to reverse-alias used in the forward phase, "
" error: %s mail_from: %s , rcpt_tos: %s , header_from: %s , header_to: %s " ,
e ,
envelope . mail_from ,
envelope . rcpt_tos ,
msg [ headers . FROM ] ,
msg [ headers . TO ] ,
)
return status . E524
2022-01-16 11:52:44 +01:00
except ( VERPReply , VERPForward ) as e :
2022-01-09 20:35:57 +01:00
LOG . w (
" email handling fail with error: %s "
" mail_from: %s , rcpt_tos: %s , header_from: %s , header_to: %s " ,
e ,
envelope . mail_from ,
envelope . rcpt_tos ,
msg [ headers . FROM ] ,
msg [ headers . TO ] ,
)
return status . E213
2022-01-07 14:57:47 +01:00
except Exception as e :
2021-06-23 19:57:21 +02:00
LOG . e (
2022-01-08 16:58:23 +01:00
" email handling fail with error: %s "
" mail_from: %s , rcpt_tos: %s , header_from: %s , header_to: %s , saved to %s " ,
2022-01-07 14:57:47 +01:00
e ,
2020-08-27 10:20:48 +02:00
envelope . mail_from ,
envelope . rcpt_tos ,
2022-01-07 14:26:58 +01:00
msg [ headers . FROM ] ,
msg [ headers . TO ] ,
2022-01-08 16:58:23 +01:00
save_email_for_debugging (
msg , file_name_prefix = e . __class__ . __name__
) , # todo: remove
2020-08-17 11:40:58 +02:00
)
2021-06-23 19:47:06 +02:00
return status . E404
2020-08-16 11:10:01 +02:00
2021-12-30 14:00:28 +01:00
@newrelic.agent.background_task ( )
2022-01-07 14:26:58 +01:00
def _handle ( self , envelope : Envelope , msg : Message ) :
2020-09-30 11:05:21 +02:00
start = time . time ( )
2021-05-06 17:20:33 +02:00
# generate a different message_id to keep track of an email lifecycle
message_id = str ( uuid . uuid4 ( ) )
set_message_id ( message_id )
2022-01-05 16:26:31 +01:00
LOG . d ( " ====>=====>====>====>====>====>====>====> " )
2021-03-16 09:17:23 +01:00
LOG . i (
2022-01-05 16:26:31 +01:00
" New message, mail from %s , rctp tos %s " ,
2020-09-30 11:05:21 +02:00
envelope . mail_from ,
envelope . rcpt_tos ,
)
2022-01-08 00:11:16 +01:00
newrelic . agent . record_custom_metric (
" Custom/nb_rcpt_tos " , len ( envelope . rcpt_tos )
)
2019-12-17 20:43:31 +01:00
2021-10-26 10:52:28 +02:00
with create_light_app ( ) . app_context ( ) :
2022-01-07 14:26:58 +01:00
ret = handle ( envelope , msg )
2021-10-13 10:27:59 +02:00
elapsed = time . time ( ) - start
2021-12-29 16:30:12 +01:00
2022-01-17 14:42:27 +01:00
LOG . i (
2021-12-30 10:24:57 +01:00
" Finish mail_from %s , rcpt_tos %s , takes %s seconds <<=== " ,
2021-10-13 10:27:59 +02:00
envelope . mail_from ,
envelope . rcpt_tos ,
elapsed ,
)
2021-12-30 14:00:28 +01:00
newrelic . agent . record_custom_metric ( " Custom/email_handler_time " , elapsed )
newrelic . agent . record_custom_metric ( " Custom/number_incoming_email " , 1 )
2021-10-13 10:27:59 +02:00
return ret
2019-11-07 17:49:26 +01:00
2020-09-30 11:05:21 +02:00
def main ( port : int ) :
""" Use aiosmtpd Controller """
controller = Controller ( MailHandler ( ) , hostname = " 0.0.0.0 " , port = port )
2020-09-02 17:36:11 +02:00
2020-09-30 11:05:21 +02:00
controller . start ( )
LOG . d ( " Start mail controller %s %s " , controller . hostname , controller . port )
2020-09-02 17:36:11 +02:00
2020-09-30 11:05:21 +02:00
if LOAD_PGP_EMAIL_HANDLER :
2021-03-16 09:17:23 +01:00
LOG . w ( " LOAD PGP keys " )
2021-10-12 14:47:01 +02:00
load_pgp_public_keys ( )
2020-09-30 11:05:21 +02:00
while True :
time . sleep ( 2 )
if __name__ == " __main__ " :
parser = argparse . ArgumentParser ( )
parser . add_argument (
" -p " , " --port " , help = " SMTP port to listen for " , type = int , default = 20381
)
args = parser . parse_args ( )
2021-03-16 09:17:23 +01:00
LOG . i ( " Listen for port %s " , args . port )
2020-09-30 11:05:21 +02:00
main ( port = args . port )