From e8013f8e0cc4000ad16634465df3ca5757379a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 17 Mar 2022 19:03:36 +0100 Subject: [PATCH] Initial parse of rpamd extra headers --- app/email/headers.py | 1 + app/email_utils.py | 5 +++ tests/example_emls/double_queue_id_header.eml | 42 +++++++++++++++++++ tests/example_emls/gmail_spoof.eml | 41 ++++++++++++++++++ tests/test_email_handler.py | 11 +++++ tests/test_email_utils.py | 5 ++- tests/utils.py | 9 ++++ 7 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 tests/example_emls/double_queue_id_header.eml create mode 100644 tests/example_emls/gmail_spoof.eml diff --git a/app/email/headers.py b/app/email/headers.py index d639272c..82315fa3 100644 --- a/app/email/headers.py +++ b/app/email/headers.py @@ -12,6 +12,7 @@ CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding" MIME_VERSION = "Mime-Version" REPLY_TO = "Reply-To" RECEIVED = "Received" +RSPAM_QUEUE_ID = "X-Rspamd-Queue-Id" CC = "Cc" DKIM_SIGNATURE = "DKIM-Signature" X_SPAM_STATUS = "X-Spam-Status" diff --git a/app/email_utils.py b/app/email_utils.py index e1f0367e..34fc56fb 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1369,6 +1369,11 @@ def sl_sendmail( def get_queue_id(msg: Message) -> Optional[str]: """Get the Postfix queue-id from a message""" + header_values = msg.get_all(headers.RSPAM_QUEUE_ID) + if header_values: + #Get last in case somebody tries to inject a header + return header_values[-1] + received_header = str(msg[headers.RECEIVED]) if not received_header: return diff --git a/tests/example_emls/double_queue_id_header.eml b/tests/example_emls/double_queue_id_header.eml new file mode 100644 index 00000000..1a576750 --- /dev/null +++ b/tests/example_emls/double_queue_id_header.eml @@ -0,0 +1,42 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: wehrman_mannequin@sldev.ovh +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: INVALIDVALUE +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_POLICY_SOFTFAIL(0.10)[gmail.com : No valid SPF, No valid DKIM,none]; + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + FROM_EQ_ENVFROM(0.00)[]; + ASN(0.00)[asn:16276, ipnet:34.59.0.0/16, country:FR]; + R_DKIM_NA(0.00)[]; + RCVD_COUNT_ZERO(0.00)[0]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + FROM_NO_DN(0.00)[]; + R_SPF_SOFTFAIL(0.00)[~all]; + FORCE_ACTION_SL_SPF_FAIL_ADD_HEADER(0.00)[add header]; + RCPT_COUNT_ONE(0.00)[1]; + FREEMAIL_FROM(0.00)[gmail.com]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] +X-Rspamd-Pre-Result: action=add header; + module=force_actions; + unknown reason +X-Spam: Yes + +This is a test mailing diff --git a/tests/example_emls/gmail_spoof.eml b/tests/example_emls/gmail_spoof.eml new file mode 100644 index 00000000..06f1db2f --- /dev/null +++ b/tests/example_emls/gmail_spoof.eml @@ -0,0 +1,41 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: wehrman_mannequin@sldev.ovh +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_POLICY_SOFTFAIL(0.10)[gmail.com : No valid SPF, No valid DKIM,none]; + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + FROM_EQ_ENVFROM(0.00)[]; + ASN(0.00)[asn:16276, ipnet:34.59.0.0/16, country:FR]; + R_DKIM_NA(0.00)[]; + RCVD_COUNT_ZERO(0.00)[0]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + FROM_NO_DN(0.00)[]; + R_SPF_SOFTFAIL(0.00)[~all]; + FORCE_ACTION_SL_SPF_FAIL_ADD_HEADER(0.00)[add header]; + RCPT_COUNT_ONE(0.00)[1]; + FREEMAIL_FROM(0.00)[gmail.com]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] +X-Rspamd-Pre-Result: action=add header; + module=force_actions; + unknown reason +X-Spam: Yes + +This is a test mailing diff --git a/tests/test_email_handler.py b/tests/test_email_handler.py index bea243ba..4d8ee815 100644 --- a/tests/test_email_handler.py +++ b/tests/test_email_handler.py @@ -1,3 +1,5 @@ +import email +import os.path from email.message import EmailMessage from app.email import headers @@ -7,6 +9,7 @@ from email_handler import ( should_ignore, is_automatic_out_of_office, ) +from tests.utils import load_eml_file def test_get_mailbox_from_mail_from(flask_client): @@ -61,3 +64,11 @@ def test_is_automatic_out_of_office(): msg[headers.AUTO_SUBMITTED] = "auto-generated" assert is_automatic_out_of_office(msg) + +def test_process_spoofed(): + msg = load_eml_file("gmail_spoof.eml") + breakpoint() + a = msg['a'] + b=1 + c=2 + diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index dc7c41f5..b43a2fb0 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -48,7 +48,7 @@ from app.models import ( ) # flake8: noqa: E101, W191 -from tests.utils import login +from tests.utils import login, load_eml_file def test_get_email_domain_part(): @@ -747,6 +747,9 @@ def test_get_queue_id(): assert get_queue_id(msg) == "4FxQmw1DXdz2vK2" +def test_get_queue_id_from_double_header(): + msg = load_eml_file("double_queue_id_header.eml") + assert get_queue_id(msg) == "6D8C13F069" def test_should_ignore_bounce(flask_client): assert not should_ignore_bounce("not-exist") diff --git a/tests/utils.py b/tests/utils.py index 5de1a4a4..1c6c4d50 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,4 +1,7 @@ +import email import json +import os +from email.message import EmailMessage from flask import url_for @@ -41,3 +44,9 @@ def create_user(flask_client) -> User: def pretty(d): """pretty print as json""" print(json.dumps(d, indent=2)) + + +def load_eml_file(filename: str) -> EmailMessage: + emails_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),"example_emls") + fullpath = os.path.join(emails_dir, filename) + return email.message_from_file(open(fullpath))