From f03bde1d8ddb7ae9b9687bfebba27259340d9e41 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Fri, 11 Dec 2020 11:02:52 +0100 Subject: [PATCH 1/5] remove _MESSAGE_ID --- email_handler.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/email_handler.py b/email_handler.py index 6b13a0b6..041d0306 100644 --- a/email_handler.py +++ b/email_handler.py @@ -129,7 +129,6 @@ _DIRECTION = "X-SimpleLogin-Type" _IP_HEADER = "X-SimpleLogin-Client-IP" _EMAIL_LOG_ID_HEADER = "X-SimpleLogin-EmailLog-ID" -_MESSAGE_ID = "Message-ID" _ENVELOPE_FROM = "X-SimpleLogin-Envelope-From" _MIME_HEADERS = [ @@ -687,7 +686,7 @@ def forward_email_to_mailbox( delete_header(msg, _IP_HEADER) add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id)) - add_or_replace_header(msg, _MESSAGE_ID, make_msgid(str(email_log.id), EMAIL_DOMAIN)) + add_or_replace_header(msg, "Message-ID", make_msgid(str(email_log.id), EMAIL_DOMAIN)) add_or_replace_header(msg, _ENVELOPE_FROM, envelope.mail_from) if not msg["Date"]: @@ -937,7 +936,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): add_or_replace_header( msg, - _MESSAGE_ID, + "Message-ID", make_msgid(str(email_log.id), get_email_domain_part(alias.email)), ) date_header = formatdate() @@ -1634,7 +1633,7 @@ def handle(envelope: Envelope) -> str: async def get_spam_score_async(message: Message) -> float: - LOG.debug("get spam score for %s", message[_MESSAGE_ID]) + LOG.debug("get spam score for %s", message["Message-ID"]) sa_input = to_bytes(message) # Spamassassin requires to have an ending linebreak From 93503d4cd3d55b029c9f664123a258e9cda99db9 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Fri, 11 Dec 2020 11:03:52 +0100 Subject: [PATCH 2/5] Do not rely on revert to delete EmailLog object when pgp fails --- email_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/email_handler.py b/email_handler.py index 041d0306..25221c97 100644 --- a/email_handler.py +++ b/email_handler.py @@ -841,6 +841,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): is_reply=True, user_id=contact.user_id, mailbox_id=mailbox.id, + commit=True ) # Spam check @@ -909,11 +910,11 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): "Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user ) # to not save the email_log - db.session.rollback() + EmailLog.delete(email_log.id) + db.session.commit() # return 421 so the client can retry later return False, "421 SL E13 Retry later" - # save the email_log to DB db.session.commit() # make the email comes from alias From c1ad161db7f540b35c7735851aca17e2978ce822 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Fri, 11 Dec 2020 11:05:01 +0100 Subject: [PATCH 3/5] add email_log to get_spam_score --- email_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/email_handler.py b/email_handler.py index 25221c97..992efc32 100644 --- a/email_handler.py +++ b/email_handler.py @@ -616,7 +616,7 @@ def forward_email_to_mailbox( if SPAMASSASSIN_HOST: start = time.time() - spam_score = get_spam_score(msg) + spam_score = get_spam_score(msg, email_log) LOG.d( "%s -> %s - spam score %s in %s seconds", contact, @@ -851,7 +851,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): # do not use user.max_spam_score here if SPAMASSASSIN_HOST: start = time.time() - spam_score = get_spam_score(msg) + spam_score = get_spam_score(msg, email_log) LOG.d( "%s -> %s - spam score %s in %s seconds", alias, @@ -1657,8 +1657,8 @@ async def get_spam_score_async(message: Message) -> float: return -999 -def get_spam_score(message: Message) -> float: - LOG.debug("get spam score for %s", message[_MESSAGE_ID]) +def get_spam_score(message: Message, email_log: EmailLog) -> float: + LOG.debug("get spam score for %s", email_log) sa_input = to_bytes(message) # Spamassassin requires to have an ending linebreak From b2e16827047c4e61e3f43dc4cd67daac04afc1ad Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Fri, 11 Dec 2020 11:12:38 +0100 Subject: [PATCH 4/5] do not override message-id in forward phase --- email_handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 992efc32..1291c7e7 100644 --- a/email_handler.py +++ b/email_handler.py @@ -686,7 +686,12 @@ def forward_email_to_mailbox( delete_header(msg, _IP_HEADER) add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id)) - add_or_replace_header(msg, "Message-ID", make_msgid(str(email_log.id), EMAIL_DOMAIN)) + + # fill up the message-id if ever it's absent. Should never happen for a normal email + if not msg["Message-ID"]: + LOG.exception("Set Message-ID before forwarding email") + msg["Message-ID"] = make_msgid(str(email_log.id), EMAIL_DOMAIN) + add_or_replace_header(msg, _ENVELOPE_FROM, envelope.mail_from) if not msg["Date"]: From 8d72d66d085cee704646cc3970e1db33059db42e Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Fri, 11 Dec 2020 11:13:19 +0100 Subject: [PATCH 5/5] keep References and In-Reply-To in reply phase for a correct email thread --- email_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/email_handler.py b/email_handler.py index 1291c7e7..0a0ef6c0 100644 --- a/email_handler.py +++ b/email_handler.py @@ -846,7 +846,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): is_reply=True, user_id=contact.user_id, mailbox_id=mailbox.id, - commit=True + commit=True, ) # Spam check @@ -893,6 +893,9 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): "To", "Cc", "Subject", + # References and In-Reply-To are used for keeping the email thread + "References", + "In-Reply-To", ] + _MIME_HEADERS, ) @@ -940,6 +943,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): replace_header_when_reply(msg, alias, "To") replace_header_when_reply(msg, alias, "Cc") + # Message-ID can reveal about the mailbox -> replace it add_or_replace_header( msg, "Message-ID", @@ -1639,7 +1643,6 @@ def handle(envelope: Envelope) -> str: async def get_spam_score_async(message: Message) -> float: - LOG.debug("get spam score for %s", message["Message-ID"]) sa_input = to_bytes(message) # Spamassassin requires to have an ending linebreak