From 5088604bb8a2bca3c7309c6f1a0f341c786703f2 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 10 Oct 2022 10:00:19 +0200 Subject: [PATCH] Replace reverse alias (#1335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * replace any reverse alias by real address for all contacts * improve logging * fix comment * Request contacts in batches of 100 to avoid loading the db * Fix typo * Added tests for the contact replacement * Increase batch size to 1k * Revert and use only reply_email and website_email Co-authored-by: Adrià Casajús --- app/config.py | 11 ++++ email_handler.py | 26 ++++++++ .../replacement_on_reply_phase.eml | 64 +++++++++++++++++++ tests/test.env | 4 +- tests/test_email_handler.py | 44 ++++++++++++- 5 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 tests/example_emls/replacement_on_reply_phase.eml diff --git a/app/config.py b/app/config.py index 99aebc5d..4e89537b 100644 --- a/app/config.py +++ b/app/config.py @@ -503,3 +503,14 @@ if not RECOVERY_CODE_HMAC_SECRET or len(RECOVERY_CODE_HMAC_SECRET) < 16: raise RuntimeError( "Please define RECOVERY_CODE_HMAC_SECRET in your configuration with a random string at least 16 chars long" ) + +# run over all reverse alias for an alias and replace them with sender address +ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT = ( + "ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT" in os.environ +) + +if ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT: + # max number of reverse alias that can be replaced + MAX_NB_REVERSE_ALIAS_REPLACEMENT = int( + os.environ["MAX_NB_REVERSE_ALIAS_REPLACEMENT"] + ) diff --git a/email_handler.py b/email_handler.py index 9d8b5233..25f32a40 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1132,8 +1132,34 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): # as this is usually included when replying if user.replace_reverse_alias: LOG.d("Replace reverse-alias %s by contact email %s", reply_email, contact) + msg = replace(msg, reply_email, contact.website_email) + if config.ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT: + start = time.time() + # MAX_NB_REVERSE_ALIAS_REPLACEMENT is there to limit potential attack + contact_query = ( + Contact.query() + .filter(Contact.alias_id == alias.id) + .limit(config.MAX_NB_REVERSE_ALIAS_REPLACEMENT) + ) + + # replace reverse alias by real address for all contacts + for (reply_email, website_email) in contact_query.values( + Contact.reply_email, Contact.website_email + ): + msg = replace(msg, reply_email, website_email) + + elapsed = time.time() - start + LOG.d( + "Replace reverse alias by real address for %s contacts takes %s seconds", + contact_query.count(), + elapsed, + ) + newrelic.agent.record_custom_metric( + "Custom/reverse_alias_replacement_time", elapsed + ) + # create PGP email if needed if contact.pgp_finger_print and user.is_premium(): LOG.d("Encrypt message for contact %s", contact) diff --git a/tests/example_emls/replacement_on_reply_phase.eml b/tests/example_emls/replacement_on_reply_phase.eml new file mode 100644 index 00000000..9fb9755a --- /dev/null +++ b/tests/example_emls/replacement_on_reply_phase.eml @@ -0,0 +1,64 @@ +Received: by mail-ed1-f49.google.com with SMTP id ej4so13657316edb.7 + for ; Mon, 27 Jun 2022 08:48:15 -0700 (PDT) +X-Gm-Message-State: AJIora8exR9DGeRFoKAtjzwLtUpH5hqx6Zt3tm8n4gUQQivGQ3fELjUV + yT7RQIfeW9Kv2atuOcgtmGYVU4iQ8VBeLmK1xvOYL4XpXfrT7ZrJNQ== +Authentication-Results: mx.google.com; + dkim=pass header.i=@matera.eu header.s=fnt header.b=XahYMey7; + dkim=pass header.i=@sendgrid.info header.s=smtpapi header.b="QOCS/yjt"; + spf=pass (google.com: domain of bounces+14445963-ab4e-csyndic.quartz=gmail.com@front-mail.matera.eu designates 168.245.4.42 as permitted sender) smtp.mailfrom="bounces+14445963-ab4e-csyndic.quartz=gmail.com@front-mail.matera.eu"; + dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=matera.eu +Received: from out.frontapp.com (unknown) + by geopod-ismtpd-3-0 (SG) + with ESMTP id d2gM2N7PT7W8d2-UEC4ESA + for ; + Mon, 27 Jun 2022 15:48:11.014 +0000 (UTC) +Content-Type: multipart/alternative; + boundary="----sinikael-?=_1-16563448907660.10629093370416887" +In-Reply-To: + +References: + + + + + +To: {{ contact_reply_email }} +Subject: Something +Message-ID: +X-Mailer: Front (1.0; +https://frontapp.com; + +msgid=af07e94a66ece6564ae30a2aaac7a34c@frontapp.com) +X-Feedback-ID: 14445963:SG +X-SG-EID: + =?us-ascii?Q?XtlxQDg5i3HqMzQY2Upg19JPZBVl1RybInUUL2yta9uBoIU4KU1FMJ5DjWrz6g?= + =?us-ascii?Q?fJUK5Qmneg2uc46gwp5BdHdp6Foaq5gg3xJriv3?= + =?us-ascii?Q?9OA=2FWRifeylU9O+ngdNbOKXoeJAkROmp2mCgw9x?= + =?us-ascii?Q?uud+EclOT9mYVtbZsydOLLm6Y2PPswQl8lnmiku?= + =?us-ascii?Q?DAhkG15HTz2FbWGWNDFb7VrSsN5ddjAscr6sIHw?= + =?us-ascii?Q?S48R5fnXmfhPbmlCgqFjr0FGphfuBdNAt6z6w8a?= + =?us-ascii?Q?o9u1EYDIX7zWHZ+Tr3eyw=3D=3D?= +X-SG-ID: + =?us-ascii?Q?N2C25iY2uzGMFz6rgvQsb8raWjw0ZPf1VmjsCkspi=2FI9PhcvqXQTpKqqyZkvBe?= + =?us-ascii?Q?+2RscnQ4WPkA+BN1vYgz1rezTVIqgp+rlWrKk8o?= + =?us-ascii?Q?HoB5dzpX6HKWtWCVRi10zwlDN1+pJnySoIUrlaT?= + =?us-ascii?Q?PA2aqQKmMQbjTl0CUAFryR8hhHcxdS0cQowZSd7?= + =?us-ascii?Q?XNjJWLvCGF7ODwg=2FKr+4yRE8UvULS2nrdO2wWyQ?= + =?us-ascii?Q?AiFHdPdZsRlgNomEo=3D?= +X-Spamd-Result: default: False [-2.00 / 13.00]; + ARC_ALLOW(-1.00)[google.com:s=arc-20160816:i=1]; + MIME_GOOD(-0.10)[multipart/alternative,text/plain]; + REPLYTO_ADDR_EQ_FROM(0.00)[]; + FORGED_RECIPIENTS_FORWARDING(0.00)[]; + NEURAL_HAM(-0.00)[-0.981]; + FREEMAIL_TO(0.00)[gmail.com]; + RCVD_TLS_LAST(0.00)[]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + MIME_TRACE(0.00)[0:+,1:+,2:~]; + RWL_MAILSPIKE_POSSIBLE(0.00)[209.85.208.49:from] + +------sinikael-?=_1-16563448907660.10629093370416887 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +Contact is {{ contact_reply_email }} +Other contact is {{ other_contact_reply_email }} +------sinikael-?=_1-16563448907660.10629093370416887-- diff --git a/tests/test.env b/tests/test.env index e942078b..68c81e40 100644 --- a/tests/test.env +++ b/tests/test.env @@ -63,4 +63,6 @@ PROTON_BASE_URL=https://localhost/api POSTMASTER=postmaster@test.domain -RECOVERY_CODE_HMAC_SECRET=1234567890123456789 \ No newline at end of file +RECOVERY_CODE_HMAC_SECRET=1234567890123456789 +ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT=true +MAX_NB_REVERSE_ALIAS_REPLACEMENT=200 \ No newline at end of file diff --git a/tests/test_email_handler.py b/tests/test_email_handler.py index d42123f7..f38ab80c 100644 --- a/tests/test_email_handler.py +++ b/tests/test_email_handler.py @@ -11,6 +11,7 @@ from app.config import EMAIL_DOMAIN, ALERT_DMARC_FAILED_REPLY_PHASE from app.db import Session from app.email import headers, status from app.email_utils import generate_verp_email +from app.mail_sender import mail_sender from app.models import ( Alias, AuthorizedAddress, @@ -26,7 +27,7 @@ from email_handler import ( should_ignore, is_automatic_out_of_office, ) -from tests.utils import load_eml_file, create_new_user +from tests.utils import load_eml_file, create_new_user, random_email def test_get_mailbox_from_mail_from(flask_client): @@ -266,3 +267,44 @@ def test_references_header(flask_client): envelope.rcpt_tos = [alias.email] result = email_handler.handle(envelope, msg) assert result == status.E200 + + +@mail_sender.store_emails_test_decorator +def test_replace_contacts_and_user_in_reply_phase(flask_client): + user = create_new_user() + user.replace_reverse_alias = True + alias = Alias.create_new_random(user) + Session.flush() + contact = Contact.create( + user_id=user.id, + alias_id=alias.id, + website_email=random_email(), + reply_email=f"{random.random()}@{EMAIL_DOMAIN}", + commit=True, + ) + contact_real_mail = contact.website_email + contact2 = Contact.create( + user_id=user.id, + alias_id=alias.id, + website_email=random_email(), + reply_email=f"{random.random()}@{EMAIL_DOMAIN}", + commit=True, + ) + contact2_real_mail = contact2.website_email + msg = load_eml_file( + "replacement_on_reply_phase.eml", + { + "contact_reply_email": contact.reply_email, + "other_contact_reply_email": contact2.reply_email, + }, + ) + envelope = Envelope() + envelope.mail_from = alias.mailbox.email + envelope.rcpt_tos = [contact.reply_email] + result = email_handler.handle(envelope, msg) + assert result == status.E200 + sent_mails = mail_sender.get_stored_emails() + assert len(sent_mails) == 1 + payload = sent_mails[0].msg.get_payload()[0].get_payload() + assert payload.find("Contact is {}".format(contact_real_mail)) > -1 + assert payload.find("Other contact is {}".format(contact2_real_mail)) > -1