From a2f141d3ccba06995d72e690cddf54d489c57b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Tue, 10 May 2022 17:54:51 +0200 Subject: [PATCH 1/9] Get recipient address from the complaint report when possible --- app/email_utils.py | 14 ++ app/handler/provider_complaint.py | 86 ++++++-- tests/example_emls/hotmail_complaint.eml | 258 ++++++++++++++++++++++ tests/example_emls/yahoo_complaint.eml | 157 +++++++++++++ tests/handler/test_provider_complaints.py | 64 +++--- tests/test.env | 2 + 6 files changed, 524 insertions(+), 57 deletions(-) create mode 100644 tests/example_emls/hotmail_complaint.eml create mode 100644 tests/example_emls/yahoo_complaint.eml diff --git a/app/email_utils.py b/app/email_utils.py index a899ad5f..47e46e2c 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1314,6 +1314,20 @@ def should_ignore_bounce(mail_from: str) -> bool: return False +def parse_address_list(address_list: str) -> List[Tuple[str, str]]: + """ + Parse a list of email addresses from a header in the form "ab , cd " + and return a list [("ab", "ab@sd.com"),("cd", "cd@cd.com")] + """ + processed_addresses = [] + for split_address in address_list.split(","): + split_address = split_address.strip() + if not split_address: + continue + processed_addresses.append(parse_full_address(split_address)) + return processed_addresses + + def parse_full_address(full_address) -> (str, str): """ parse the email address full format and return the display name and address diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index ce2befec..c316fdba 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -1,8 +1,9 @@ import uuid from abc import ABC, abstractmethod +from dataclasses import dataclass from io import BytesIO from mailbox import Message -from typing import Optional +from typing import Optional, List from app import s3 from app.config import ( @@ -18,6 +19,7 @@ from app.email_utils import ( to_bytes, render, send_email_with_rate_control, + parse_address_list, ) from app.log import LOG from app.models import ( @@ -33,12 +35,23 @@ from app.models import ( ) +@dataclass +class OriginalAddresses: + sender: str + recipient: str + + class ProviderComplaintOrigin(ABC): @classmethod @abstractmethod def get_original_message(cls, message: Message) -> Optional[Message]: pass + @classmethod + @abstractmethod + def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + pass + @classmethod @abstractmethod def name(cls): @@ -58,6 +71,33 @@ class ProviderComplaintYahoo(ProviderComplaintOrigin): return part return None + @classmethod + def get_feedback_report(cls, message: Message) -> Optional[Message]: + for part in message.walk(): + if part["content-type"] == "message/feedback-report": + content = part.get_payload() + if not content: + continue + return content[0] + return None + + @classmethod + def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + report = cls.get_feedback_report(message) + original = cls.get_original_message(message) + rcpt_address = report["original-rcpt-to"] + try: + if rcpt_address: + _, rcpt_address = parse_full_address(rcpt_address) + else: + rcpt_address = parse_address_list(original[headers.TO])[0] + _, sender_address = parse_full_address(original[headers.FROM]) + return OriginalAddresses(sender_address, rcpt_address) + except ValueError: + saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") + LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + return False + @classmethod def name(cls): return "yahoo" @@ -76,6 +116,22 @@ class ProviderComplaintHotmail(ProviderComplaintOrigin): return part return None + @classmethod + def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + try: + part = cls.get_original_message(message) + rcpt_address = part["x-simplelogin-envelope-to"] + if rcpt_address: + _, rcpt_address = parse_full_address(rcpt_address) + else: + rcpt_address = parse_address_list(part[headers.TO])[0] + _, sender_address = parse_full_address(part[headers.FROM]) + return OriginalAddresses(sender_address, rcpt_address) + except ValueError: + saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") + LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + return False + @classmethod def name(cls): return "hotmail" @@ -98,45 +154,35 @@ def find_alias_with_address(address: str) -> Optional[Alias]: def handle_complaint(message: Message, origin: ProviderComplaintOrigin) -> bool: - original_message = origin.get_original_message(message) - - try: - _, to_address = parse_full_address( - get_header_unicode(original_message[headers.TO]) - ) - _, from_address = parse_full_address( - get_header_unicode(original_message[headers.FROM]) - ) - except ValueError: - saved_file = save_email_for_debugging(message, "FromParseFailed") - LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + addresses = origin.get_original_addresses(message) + if not addresses: return False - user = User.get_by(email=to_address) + user = User.get_by(email=addresses.recipient) if user: LOG.d(f"Handle provider {origin.name()} complaint for {user}") report_complaint_to_user_in_transactional_phase(user, origin) return True - alias = find_alias_with_address(from_address) + alias = find_alias_with_address(addresses.sender) # the email is during a reply phase, from=alias and to=destination if alias: LOG.i( - f"Complaint from {origin.name} during reply phase {alias} -> {to_address}, {user}" + f"Complaint from {origin.name} during reply phase {alias} -> {addresses.recipient}, {user}" ) - report_complaint_to_user_in_reply_phase(alias, to_address, origin) + report_complaint_to_user_in_reply_phase(alias, addresses.recipient, origin) store_provider_complaint(alias, message) return True - contact = Contact.get_by(reply_email=from_address) + contact = Contact.get_by(reply_email=addresses.sender) if contact: alias = contact.alias else: - alias = find_alias_with_address(to_address) + alias = find_alias_with_address(addresses.recipient) if not alias: LOG.e( - f"Cannot find alias from address {to_address} or contact with reply {from_address}" + f"Cannot find alias for address {addresses.recipient} or contact with reply {addresses.sender}" ) return False diff --git a/tests/example_emls/hotmail_complaint.eml b/tests/example_emls/hotmail_complaint.eml new file mode 100644 index 00000000..a3fbeda7 --- /dev/null +++ b/tests/example_emls/hotmail_complaint.eml @@ -0,0 +1,258 @@ +X-SimpleLogin-Client-IP: 40.92.66.13 +Received-SPF: Pass (mailfrom) identity=mailfrom; client-ip=40.92.66.13; + helo=eur01-ve1-obe.outbound.protection.outlook.com; + envelope-from=staff@hotmail.com; receiver= +Received: from EUR01-VE1-obe.outbound.protection.outlook.com + (mail-oln040092066013.outbound.protection.outlook.com [40.92.66.13]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by prod4.simplelogin.co (Postfix) with ESMTPS id 408E09C472 + for <{{ postmaster }}>; Mon, 9 May 2022 13:11:34 +0000 (UTC) +ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none; + b=V3N8KdYGgYrjs5KcjFUA0MgPUmOc+NV4ygLfSd7fehfiNemKdhe6Cpfj58zWFNzoG5qBoUCIm/BI7aCr7lqAU2hQJypTrJG+3zbSdnuCKMBVV5GHZxkE+XAeSU+4wt4xwl1ZiVx/2P//xUVWN/TVmiuKUgCn9n+WagU9LYGVT9z6wwOpXggpDf6ow9RnJDPJpkakHRh7rQPABbrOpVqEZnoJdAH5mgdTHJOeBumNym4i3GKnky+IfMlqwGcbTrzgrt/D3PpZdsMG4B+jEHtTo3FgB9JY+abjU9Bvn4rXwKr3RMF+1ZV3UsznQVwuT99PtfEcExV3zSsqEPDBy9QT9w== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; + s=arcselector9901; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1; + bh=Y37p6EaXY5hpBNgMr1ILYzy35GKdkqWXm69FR2RyQgA=; + b=aet1P4fpmUM9bqbLD3vtp/EWfUi2WfvWbOnnLg/YZ2vxoTF/eM5IHDBB/I7btdzZICric+KkhRih/kvaVURGy4jybYjn9FNfT+HShTJa75Pk30fp3in/5lL2x6Q0xM0Naf9YtTvGgqlLDrdgCmktxyByNAOFPo27fEWy3fk/00IPWyI8j77VvYsGn8rJCLbhDUBWwGzQ9P7SabIqn9Ybx6CKcw2FssJhSNAyOIx7EkrGxq8y/5dXeWSHLFBdHPu6F9w/DKyt9cv17rBSnHo4tx1Ese93vBHT5XIwTwnGisCa0++eqL/69GugKoe5odkAfsdRAlBjVTgXp2Lol4rrpg== +ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none; + dkim=none; arc=none +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=hotmail.com; + s=selector1; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=Y37p6EaXY5hpBNgMr1ILYzy35GKdkqWXm69FR2RyQgA=; + b=uMkd90Lx6ikNpk7RRBU3AfQ0jjbjRZAGQLnY3r+dQ3CNnhgfHxpNRudxGDydmf6GQ2AuylmOnLVATh8XMKTvCnVg8hjB9xrxd5qPpQ3k92U5VlgVe1o1Nwq8R6VCJugOZduDjSJdBXO2ACosUul6IQXKMBpSNq+bGJ9VHu63EGTphkWOOw1a4PArg8tQTSmkpkyh788nsfNXnVsh2fkL6we1LyvagQzTS4e1ynuSk1zAk+6U5KOuhRVr2Nh/AvyvswWpjA4pflOqFwyqsMYb3N6wnpRTct8CJUPlQwEx6chiJgKNGrAkdRbnWaEyeIEdyJB/NLwtPqZzKYFgv7f8wg== +Received: from AM6PR02CA0021.eurprd02.prod.outlook.com (2603:10a6:20b:6e::34) + by AM0PR02MB4563.eurprd02.prod.outlook.com (2603:10a6:208:ec::33) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.22; Mon, 9 May + 2022 13:11:32 +0000 +Received: from AM6EUR05FT047.eop-eur05.prod.protection.outlook.com + (2603:10a6:20b:6e:cafe::26) by AM6PR02CA0021.outlook.office365.com + (2603:10a6:20b:6e::34) with Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.23 via Frontend + Transport; Mon, 9 May 2022 13:11:32 +0000 +Received: from DM5SVC01SF077 (40.107.211.126) by + AM6EUR05FT047.mail.protection.outlook.com (10.233.241.167) with Microsoft + SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id + 15.20.5227.15 via Frontend Transport; Mon, 9 May 2022 13:11:32 +0000 +X-IncomingTopHeaderMarker: + OriginalChecksum:86053024C4DD515561A96BAF61AACB6F8A4DB30C8D14CAC5F2F7D189ACDCA109;UpperCasedChecksum:5323AB267D58619B82076460438A30DFDD8E7969870D76B723156F921928319B;SizeAsReceived:257;Count:6 +Date: Mon, 9 May 2022 13:10:08 +0000 +From: +Subject: complaint about message from 176.119.200.162 +To: {{ postmaster }} +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="31A9507D-D0B3-4DCD-AFBB-413468892CFE" +X-IncomingHeaderCount: 6 +Message-ID: + <1d63d9ee-8f3e-4876-955c-1807db5ad138@AM6EUR05FT047.eop-eur05.prod.protection.outlook.com> +X-EOPAttributedMessage: 0 +X-MS-PublicTrafficType: Email +X-MS-Office365-Filtering-Correlation-Id: 44e9ec0b-6c5d-4cea-6417-08da31bd7000 +X-MS-TrafficTypeDiagnostic: AM0PR02MB4563:EE_ +X-Microsoft-Antispam: BCL:0; +X-Microsoft-Antispam-Message-Info: + lK5xD4UZS47NfR0tHc3wEp4HHOifZ4SDBb8aKx7H/vEW8Rg8rXXH12G4lWdpzr8qTsCmvzuhj5x6IAumOKQ8lWLj5Lp3jyml91wVnwCtUnk5cTXpQwDZd9QMgtEW07GoLdWjkbShAhLRDf+9Y4DxidHCacOAYxcNX42wo3vYZOEHDzVRUxSmY0c7Km60pDtiYzEk+P9AoE2YKYG2rDwDx0vgoLgqFspGqQ+2OeHD2ZAEyATHR/sQy6tf5S2d4wA3HcHrwrGMlz/4d9VbT5h9a5cqj9S59wpuc6g8nyYhmK3AHJkB5nXmpBZBihTw5X/Qh5PZqUYwPxkwpq3WlaEuXvzaKFiwJFvtuRGX+mEioClCxiwPROb7sI9ZHWPw48AHysF+whYGBfleRy4c2SuW6e1D5uewGry+lXVljxg7qKo= +X-OriginatorOrg: sct-15-20-4755-11-msonline-outlook-ab7de.templateTenant +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 09 May 2022 13:11:32.0875 + (UTC) +X-MS-Exchange-CrossTenant-Network-Message-Id: + 44e9ec0b-6c5d-4cea-6417-08da31bd7000 +X-MS-Exchange-CrossTenant-Id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa +X-MS-Exchange-CrossTenant-AuthSource: + AM6EUR05FT047.eop-eur05.prod.protection.outlook.com +X-MS-Exchange-CrossTenant-AuthAs: Anonymous +X-MS-Exchange-CrossTenant-FromEntityHeader: Internet +X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: + 00000000-0000-0000-0000-000000000000 +X-MS-Exchange-Transport-CrossTenantHeadersStamped: AM0PR02MB4563 +X-Spamd-Result: default: False [-1.75 / 13.00]; + ARC_ALLOW(-1.00)[microsoft.com:s=arcselector9901:i=1]; + DMARC_POLICY_ALLOW(-0.50)[hotmail.com,none]; + R_SPF_ALLOW(-0.20)[+ip4:40.92.0.0/15]; + MIME_HTML_ONLY(0.20)[]; + R_DKIM_ALLOW(-0.20)[hotmail.com:s=selector1]; + MIME_GOOD(-0.10)[multipart/mixed,multipart/related]; + MANY_INVISIBLE_PARTS(0.05)[1]; + NEURAL_HAM(-0.00)[-0.996]; + FROM_EQ_ENVFROM(0.00)[]; + FREEMAIL_ENVFROM(0.00)[hotmail.com]; + MIME_TRACE(0.00)[0:+,1:~,2:+,3:+,4:~]; + ASN(0.00)[asn:8075, ipnet:40.80.0.0/12, country:US]; + RCVD_IN_DNSWL_NONE(0.00)[40.92.66.13:from]; + DKIM_TRACE(0.00)[hotmail.com:+]; + RCVD_TLS_LAST(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + FREEMAIL_FROM(0.00)[hotmail.com]; + FROM_NO_DN(0.00)[]; + TO_DN_NONE(0.00)[]; + RCVD_COUNT_THREE(0.00)[4]; + RCPT_COUNT_ONE(0.00)[1]; + DWL_DNSWL_NONE(0.00)[hotmail.com:dkim] +X-Rspamd-Queue-Id: 408E09C472 +X-Rspamd-Server: prod4 +Content-Transfer-Encoding: 7bit + +--31A9507D-D0B3-4DCD-AFBB-413468892CFE +Content-Type: message/rfc822 +Content-Disposition: inline + +X-HmXmrOriginalRecipient: +X-MS-Exchange-EOPDirect: true +Received: from SJ0PR11MB4958.namprd11.prod.outlook.com (2603:10b6:a03:2ae::24) + by SA0PR11MB4525.namprd11.prod.outlook.com with HTTPS; Mon, 9 May 2022 + 04:30:48 +0000 +Received: from BN9PR03CA0117.namprd03.prod.outlook.com (2603:10b6:408:fd::32) + by SJ0PR11MB4958.namprd11.prod.outlook.com (2603:10b6:a03:2ae::24) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.20; Mon, 9 May + 2022 04:30:45 +0000 +Received: from BN8NAM11FT053.eop-nam11.prod.protection.outlook.com + (2603:10b6:408:fd:cafe::d0) by BN9PR03CA0117.outlook.office365.com + (2603:10b6:408:fd::32) with Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.20 via Frontend + Transport; Mon, 9 May 2022 04:30:45 +0000 +Authentication-Results: spf=pass (sender IP is 176.119.200.162) + smtp.mailfrom=simplelogin.co; dkim=pass (signature was verified) + header.d=simplelogin.co;dmarc=pass action=none + header.from=simplelogin.co;compauth=pass reason=100 +Received-SPF: Pass (protection.outlook.com: domain of simplelogin.co + designates 176.119.200.162 as permitted sender) + receiver=protection.outlook.com; client-ip=176.119.200.162; + helo=mail-200162.simplelogin.co; +Received: from mail-200162.simplelogin.co (176.119.200.162) by + BN8NAM11FT053.mail.protection.outlook.com (10.13.177.209) with Microsoft SMTP + Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id + 15.20.5227.15 via Frontend Transport; Mon, 9 May 2022 04:30:44 +0000 +X-IncomingTopHeaderMarker: + OriginalChecksum:5EBD8C309CA888838EDC898C63E28E1EC00EF74772276A54C08DA83D658756F4;UpperCasedChecksum:E102374CD208D4ACB2034F1A17F76DA6345BD176395C6D4EADEC3B47BFF41ECC;SizeAsReceived:1262;Count:15 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=simplelogin.co; + s=dkim; t=1652070640; h=From:To:Subject:Message-ID:Date; + bh=Tu2Q0oO5GuGw4CVxDAdixtRKr6jqMWjpH9zEf50uKwg=; + b=o6I0Ij1CahU9EUj/9uwWJpsDjfi/2gQIXT0KJT6IAK9hOoJ5bVqPsqtyGTfIoqYhhtD/ic + 5NybKJmB6B6KL5hl5LG3KzCdaWfe3dAAhD4e2gIU80dal596dlzluyvLR1k+6rdM4JvlGq + OVWLR42Oj4anrnOqLCUkL44ILIhLpAE= +Date: Mon, 9 May 2022 00:30:38 -0400 (EDT) +Message-ID: + <10627474.1041327707.1652070638478.JavaMail.cloud@p2-mta-0301.p2.messagegears.net> +Subject: Original Subject +Content-Type: multipart/mixed; + boundary="----=_Part_1041327705_575167926.1652070638478" +Content-Transfer-Encoding: 7bit +X-SimpleLogin-Type: Forward +X-SimpleLogin-EmailLog-ID: 832832 +X-SimpleLogin-Envelope-To: {{ rcpt }} +From: {{ sender }} +Reply-To: {{ sender }} +To: {{ rcpt_comma_list }} +List-Unsubscribe: +X-SimpleLogin-Want-Signing: yes +X-IncomingHeaderCount: 15 +Return-Path: {{ return_path }} +X-MS-Exchange-Organization-ExpirationStartTime: 09 May 2022 04:30:45.1195 + (UTC) +X-MS-Exchange-Organization-ExpirationStartTimeReason: OriginalSubmit +X-MS-Exchange-Organization-ExpirationInterval: 1:00:00:00.0000000 +X-MS-Exchange-Organization-ExpirationIntervalReason: OriginalSubmit +X-MS-Exchange-Organization-Network-Message-Id: + ede92e41-5acb-4474-c5be-08da3174af2b +X-EOPAttributedMessage: 0 +X-EOPTenantAttributedMessage: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa:0 +X-MS-Exchange-Organization-MessageDirectionality: Incoming +X-MS-PublicTrafficType: Email +X-MS-Exchange-Organization-AuthSource: + BN8NAM11FT053.eop-nam11.prod.protection.outlook.com +X-MS-Exchange-Organization-AuthAs: Anonymous +X-MS-UserLastLogonTime: 5/9/2022 3:30:52 AM +X-MS-Office365-Filtering-Correlation-Id: ede92e41-5acb-4474-c5be-08da3174af2b +X-MS-TrafficTypeDiagnostic: SJ0PR11MB4958:EE_ +X-MS-Exchange-EOPDirect: true +X-Sender-IP: 176.119.200.162 +X-SID-PRA: PHWNQHFTTLQNZJXKMLHZCSKLLLJXMGEJOEOWW@SIMPLELOGIN.CO +X-SID-Result: PASS +X-MS-Exchange-Organization-PCL: 2 +X-MS-Exchange-Organization-SCL: 1 +X-Microsoft-Antispam: BCL:0; +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 09 May 2022 04:30:44.9945 + (UTC) +X-MS-Exchange-CrossTenant-Network-Message-Id: + ede92e41-5acb-4474-c5be-08da3174af2b +X-MS-Exchange-CrossTenant-Id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa +X-MS-Exchange-CrossTenant-AuthSource: + BN8NAM11FT053.eop-nam11.prod.protection.outlook.com +X-MS-Exchange-CrossTenant-AuthAs: Anonymous +X-MS-Exchange-CrossTenant-FromEntityHeader: Internet +X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: + 00000000-0000-0000-0000-000000000000 +X-MS-Exchange-Transport-CrossTenantHeadersStamped: SJ0PR11MB4958 +X-MS-Exchange-Transport-EndToEndLatency: 00:00:03.3271765 +X-MS-Exchange-Processed-By-BccFoldering: 15.20.5227.023 +X-Microsoft-Antispam-Mailbox-Delivery: + abwl:0;wl:0;pcwl:0;kl:0;iwl:0;ijl:0;dwl:0;dkl:0;rwl:0;ucf:0;jmr:0;ex:0;auth:1;dest:I;ENG:(5062000285)(90000117)(90005022)(91005020)(91035115)(5061607266)(5061608174)(9050020)(9100338)(2008001134)(2008000189)(2008120399)(2008019284)(2008021020)(8390246)(8377080)(8386120)(4810004)(4910013)(9910022)(9510006)(10110021)(9320005); +X-Message-Info: + 5vMbyqxGkdcvoPRAk5ACFywqndfpuBMcVz6K/12RtMALmdfGi+GpgO+lXQe3PiGwHtV5wXFRStQwg29XySZZo6tOyvshTSJ1uafhX53S93r5MaqDxJrR0UNGr2VYdKiAm1jYIYQm84v/mEbSAGjjBwEgS1PHlzM72I96JadXzfV9Fmsd5pHlfoLxEqXe6hBJAAQS99CcpwPDnaVA9UZUHA== +X-Message-Delivery: Vj0xLjE7dXM9MDtsPTA7YT0wO0Q9MTtHRD0xO1NDTD0tMQ== +X-Microsoft-Antispam-Message-Info: + =?utf-8?B?VjZIQkpKR05oRUo1Vzc0YTBDUW52S0lsYkJSMGRzY0hJMnRMOWdyRGowcGpk?= + =?utf-8?B?SUJLSDRPaStzakpJUHlaWVFnNWpBSGRsZ1Z4aEFmaXJOR1ZMUWxTTnQ1SXg1?= + =?utf-8?B?anhFNTJ5RGU2YjRiTWhWK3FvWXBJU29YSWdqM3VvUkZpY21aaW5lSkJ5WWph?= + =?utf-8?B?L2pxclptbVBGdm02emlHT3ZBQ1BHZTcrM0c3NmJ5alJLSGlaYVMvK0hwVmJV?= + =?utf-8?B?eHlTU2grSElBTVY5cXF2d250OXBmQ2pzeEVUWTlSZ1hCc1dEdStXMzFGcWlO?= + =?utf-8?B?VytUeEgyRWl5a2U1Y09VKyt3am9ZQVYrRm1LUkhRRGdKbkFTaHc4RTErQ1c0?= + =?utf-8?B?RjBNVllEVW9UakJIQm5FWWVYd2RuaENZTVJIUkI4RmlheWsyajZmanFCUlpt?= + =?utf-8?B?ZTJYZlg1RGxkbEVlRk0zallRWStiU1Z1QmJlTmtKS3J5MmZuOFk2blRHemEw?= + =?utf-8?B?OVhkUUhWWTAzV2dySnMra1pKMGo1Zy8xSFNuemx4Slg1ckhDcitmVGRHSDBW?= + =?utf-8?B?MFlOMDFtNmRPTDVSL3BGU0VNNWRObGVkUUlRcG9MSUJFeVBFcGtlVENSZmIr?= + =?utf-8?B?V3F6by8vOHBROWplTi9JdWtEVDFwUVZsdVk5djBtN0wzbk04RG56RjRsM1ZH?= + =?utf-8?B?cytsajBZNUNwUXk5SVRFZXhMejN3anYweGpCWkltQ2lwQnA3V1B6UUt0VUw1?= + =?utf-8?B?dXpLQ3hxemNQNWRGWmpqZi9BY2EzOTAwQ3h5RlF2RHQyVG1McWp6N1JXUWRY?= + =?utf-8?B?TjlCRWFmNFhQSitwSTk2cEhPK1N3ZVQxbktlMWFwa05hNGllOVpCc2Q3MUEy?= + =?utf-8?B?TlBHVE9YUE8xRUk3dndyNkFQVlhhN3JIMnUxL25pZ3JaM1hFS0VUOXNqT2NF?= + =?utf-8?B?Y3lFcUM0dDVuOGhTdmJ1RjJJK2sxZGViOUU2SE1DTUZ1c0pSSlNsazdPWHJ5?= + =?utf-8?B?TXo0dUUrZEhqaVpGTHNTUnNUTUl2L2hZeFhoNUVtcmJPQ0lXYnV5Yy8rSXBq?= + =?utf-8?B?bjYwVlBET0ErZkQ4KzJsQmM5b0hUTXJSSWlhdXlNeTZ2a0xlaHp5ZTZRQnox?= + =?utf-8?B?T2h2NkZKNmpLcDg4TCs5ckdoU3d5aEc1Q1FYUFdTOXhxcFJsaTdtZkVuNG1W?= + =?utf-8?B?SkVsN2llT3FpTnB6Q3lMbDR4ZzVzblhLVWw3VkpJblRQQVA4cDd1aGdtbll4?= + =?utf-8?B?U2RWQXplZjRreWhJRnQwWGhWT2pnVmxwTW9hdUxwRE9VaTJqd1lqenh3T2pK?= + =?utf-8?B?R2ZMaDJmNm1lS25TNU56ODFBcnc1TUZQbi9pZ0hnampKNUl0MzVQRG5wenZH?= + =?utf-8?B?dTdrcTA4VXUwZmdNaXBKMnVsY1phOEtLUEZWMzNnUlVxYXhrRDFUN3FFN0lZ?= + =?utf-8?B?MnVzbmhVQ2kvQVkzZ3NBQnNGL0NCNlZTbmV5ZW9FVWg5dUJTbmtaQnNZemRT?= + =?utf-8?B?cDFKUnRPU2VpNnNwM3V5eXJxMy9YbFhPYTRFSkEyTUZjSVlNaFV0UE5RbjhK?= + =?utf-8?B?NjJmckpva2xuaGhYT2Jkb2g1U1NEaFJmQWc5bVhheGZYMXY1b2toaVRPOXNT?= + =?utf-8?B?Y2ZhVjYyY0pnbmw4N3VneVR6bXFoRTlndE9lTzlac0JTRWFKc1BMTmNrNFMx?= + =?utf-8?B?M0lwTXI3STZXcFNmbytNcFB2VzJFSFpLSWFpbjlzcVlVRHk3RTFIUUQzOUlB?= + =?utf-8?B?YnR1eC9jUnVNWlhadktVKzM5MmdmR1pBTXVxK2xzUXZ4MzNUWW5rQXZ4SXMv?= + =?utf-8?B?RnBLUmcwT3FUWENucWtuTWhBQnl5VWFpczNGUnBkQ0ltM2ttMDM1RnFScXFa?= + =?utf-8?B?dEtNNnF4Q1FDS2RqRTRuRkNRUC9JVTdZZ216c3hycC9ZalptbDZNZ25ydWFp?= + =?utf-8?B?Z25qMGFLK1FQYm0vUU40OSt1SVJBTmdPTVNRN2JTVmxLTlRJMkZDeldKYWNx?= + =?utf-8?B?VEJEVHE5ZE9QNWsxZkxrb0pFOEU5cUJvT3ArOUFDMXlZM2N4Smk5ay9qQXEv?= + =?utf-8?B?ZXc3ZjVHMjdkcjBkN1Rodmdyd1JldkFBeDlVblRVbkxrY0xhZkIwVzBpTlNM?= + =?utf-8?B?THAvZ01hS3NVK0dHblFFQ0h6VXYydW1QaUwzM29zcjRYRFJRTU9NZWYxQ2Nw?= + =?utf-8?B?N1liQ3g2ZUtveTdTaW1ZSGovLzNWbWh2bDd6ZXRUR3B3eEYwakVCOS95aEs0?= + =?utf-8?B?NkkzL1dQREVlVHFXWmE4RktDUHFENVQwYW9YWE9LS2hrMzAyVWFXTDZFVkx5?= + =?utf-8?B?cU1nZDkzOTR1dk40SHFIcHRDSVRPajMvSVAyd0JQNDJnaVoxNmhNOFEzdzlj?= + =?utf-8?B?ODdUNXRIVkQvTHYzMytWY2o3UHZkdUNTR1pvSVJvclVCN01EZW5pVXdRUDgx?= + =?utf-8?B?Vmg2aUdlOUJzdXlPdXFlL01raHZSbkRONncyRlFLcGpLUFR4bm9BQXVJMHJC?= + =?utf-8?B?cWdJSFJwZEVkZjZkOTJqZG1FNHdZRWpGdUR6R2hjdHRoMTg1Z2lpeGpnZzlH?= + =?utf-8?B?Um5WOEJINFBFM3Evdmt4VVRCQnAwd2xBRGVralpwRnV0eUhJNTluQzFLQXI2?= + =?utf-8?B?NXI4amV3c0ZRZEZLRjE1ZEQ3aW90Y1I0K3NPN3ZoVyt1UVdzWUpQUGh1b25N?= + =?utf-8?Q?amuRKzTLQzIrlx9Vmv+SjIosxogY=3D?= +MIME-Version: 1.0 + +------=_Part_1041327705_575167926.1652070638478 +Content-Type: multipart/related; + boundary="----=_Part_1041327706_445426653.1652070638478" + +------=_Part_1041327706_445426653.1652070638478 +Content-Type: text/html;charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +Here goes the original email content + +------=_Part_1041327706_445426653.1652070638478-- + +------=_Part_1041327705_575167926.1652070638478-- + +--31A9507D-D0B3-4DCD-AFBB-413468892CFE-- diff --git a/tests/example_emls/yahoo_complaint.eml b/tests/example_emls/yahoo_complaint.eml new file mode 100644 index 00000000..a543024a --- /dev/null +++ b/tests/example_emls/yahoo_complaint.eml @@ -0,0 +1,157 @@ +X-SimpleLogin-Client-IP: 66.163.186.21 +Received-SPF: None (mailfrom) identity=mailfrom; client-ip=66.163.186.21; + helo=sonic326-46.consmr.mail.ne1.yahoo.com; + envelope-from=feedback@arf.mail.yahoo.com; receiver= +Received: from sonic326-46.consmr.mail.ne1.yahoo.com + (sonic326-46.consmr.mail.ne1.yahoo.com [66.163.186.21]) + (using TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 (128/128 bits) + key-exchange ECDHE (P-256) server-signature RSA-PSS (2048 bits) + server-digest SHA256) + (No client certificate requested) + by prod4.simplelogin.co (Postfix) with ESMTPS id 160E19C47C + for <{{ postmaster }}>; Sun, 8 May 2022 13:31:32 +0000 (UTC) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arf.mail.yahoo.com; + s=arf; t=1652016690; bh=y3TXlG8d2nUmz+Mm6gBEX1p1y2rwlM+LRC89Bp+HwGo=; + h=Date:From:To:Subject:From:Subject:Reply-To; + b=HyuY58LSzfkdH9FynjNWEl6QJeeImKRbIzrnR64sY/ggFD6fF9w1/fpXDmJ8RHpB/72llGb8nkVJkn/TK+adBCZvw4Y0SC2m8qbn6BdaC5kvAWkN6VUxvQWFMWTptAmeX+UUxY2hjEXLZQwNUd4nvvhZkbdyzw5wFSpYX0hnxAA= +X-SONIC-DKIM-SIGN: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=s2048; + t=1652016690; bh=0SlXAOx+1D8SxkBJpASrTwUGjphtzchFZOSJr0X+U2m=; + h=X-Sonic-MF:Date:From:To:Subject:From:Subject; + b=smqcDrz5jxsmGycWk9tNncLBjcQIqBnZmsQzkJ6g8fyhQw2e30y05iTnsOBTr0S9qTPK3I2JBv0P73TH7vDAnZAnaewzj9Dymw7Z+UxXKdrPBf/tD8RGw9cX6C0eb7GUjHvbvXS03IkSGnvOPPCXLsTDXYOTflcU7A0A2L+cS9ogEBl/4AFwBf/z+lcMH20h2dZ6+wPtqPCgRY1Hf45cv4gfHrFG0a18n3BBq0doCA4cRTXeeuv06fqsUCk2GF6z0mm3YWu+umcUs16QmgjHKhy4SJHvTZfx4zFBxQEOM3hvBzriL5g0D3Rg71CdkI8TVqsyXS1YWVSQFakAw0hM+A== +X-Sonic-MF: feedback@arf.mail.yahoo.com +Received: from sonic.gate.mail.ne1.yahoo.com by + sonic326.consmr.mail.ne1.yahoo.com with HTTP; Sun, 8 May 2022 13:31:30 +0000 +Date: Sun, 8 May 2022 13:31:28 +0000 (UTC) +From: Yahoo! Mail AntiSpam Feedback +To: {{ postmaster }} +Message-ID: + <1486688083.18136997.1652016688605@chakraconsumer2.asd.mail.ne1.yahoo.com> +Subject: Original subject +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=feedback-report; + boundary="----=_Part_18136996_1734597748.1652016688604" +X-Yahoo-Newman-Property: cfl +X-Yahoo-Newman-Id: cfl-test +X-Spamd-Result: default: False [-0.65 / 13.00]; + DMARC_POLICY_ALLOW(-0.50)[yahoo.com,reject]; + R_DKIM_ALLOW(-0.20)[arf.mail.yahoo.com:s=arf]; + SUBJ_ALL_CAPS(0.15)[2]; + MIME_GOOD(-0.10)[text/plain,multipart/alternative]; + R_SPF_NA(0.00)[no SPF record]; + FROM_EQ_ENVFROM(0.00)[]; + MIME_TRACE(0.00)[0:~,1:+,2:~,3:+,4:~,5:+,6:+,7:~]; + RCVD_TLS_LAST(0.00)[]; + RCVD_IN_DNSWL_NONE(0.00)[66.163.186.21:from]; + ASN(0.00)[asn:36646, ipnet:66.163.184.0/21, country:US]; + ARC_NA(0.00)[]; + DKIM_TRACE(0.00)[arf.mail.yahoo.com:+]; + MID_RHS_MATCH_FROMTLD(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + FROM_HAS_DN(0.00)[]; + RCVD_COUNT_TWO(0.00)[2]; + TO_DN_NONE(0.00)[]; + RCPT_COUNT_ONE(0.00)[1]; + NEURAL_SPAM(0.00)[0.429]; + DWL_DNSWL_NONE(0.00)[yahoo.com:dkim] +X-Rspamd-Queue-Id: 160E19C47C +X-Rspamd-Server: prod4 +Content-Transfer-Encoding: 7bit + +------=_Part_18136996_1734597748.1652016688604 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +This is an email abuse report for an email message from simplelogin.co on Sun, 8 May 2022 11:12:35 +0000 + +------=_Part_18136996_1734597748.1652016688604 +Content-Type: message/feedback-report +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Feedback-Type: abuse +User-Agent: Yahoo!-Mail-Feedback/2.0 +Version: 0.1 +Original-Mail-From: + <{{ return_path }}> +Original-Rcpt-To: {{ rcpt }} +Received-Date: Sun, 8 May 2022 11:12:35 +0000 +Reported-Domain: simplelogin.co +Authentication-Results: authentication result string is not available + + +------=_Part_18136996_1734597748.1652016688604 +Content-Type: message/rfc822 +Content-Disposition: inline + +Received: from 10.217.151.74 + by atlas316.free.mail.ne1.yahoo.com with HTTPS; + Sun, 8 May 2022 11:12:34 +0000 +Return-Path: + <{{ return_path }}> +X-Originating-Ip: [176.129.238.160] +Received-SPF: pass (domain of simplelogin.co designates 176.119.200.160 as + permitted sender) +Authentication-Results: atlas316.free.mail.ne1.yahoo.com; + dkim=pass header.i=@simplelogin.co header.s=dkim; + spf=pass smtp.mailfrom=simplelogin.co; + dmarc=pass(p=QUARANTINE) header.from=simplelogin.co; +X-Apparently-To: syn_flood91@yahoo.com; Sun, 8 May 2022 11:12:35 +0000 +X-YMailISG: 5XbMksQWLDvXV9CBjagtqIT6OTC44ku5XiuZJQp_W6hhWfR. + .wUIhFV6vRR_JeMUxC0ZAvugteAP2pe.bqk06ovvYnhJMg_HTvcmfVltbWxQ + tK7xNSs8D2PWQdyDDzB3rdFdIIfSrQnDTGjP2xpTAqLQk3IXSuUBX7s4f8uA + WUELPWj36_Xtqrwyj.ya4Ezw_ePzPhZGmMdCsbz2H5Jh45TLbk5HhL.TDDbH + 9Dz__HKLUC8acH0hu1vrPvo1ljzwbl_0cqlj10qMIChpB51XVDtyNA_WgWvE + QL1hFHS0tScfRT0xATM8w8FJv1eA0ODjakDtTRgmaWBTphzeoR.FyTBj14y5 + burx6lkUqipfP7UZpNmcNDYHQdTEmdGa8JDZMX.lpM5IMOhkByIQuoTN4.Cx + 8qz9kb.o0DqxqNRgn4_fRRAoSn1xejDbzZMu.SWSvJ1KJwAfLtep37ISqNKl + yeBeDJFMnHUjRD8B2wBB46zq4ngHFWjBGkAGQVBssLzj594FXg13aO.TnJU7 + WJ_cUSzoaH9HjgYDTi4.1x68jVxpZIEdhDe7pjLCUL2ugWdar9S7pFlyKWfa + iTH8yQ10NXtLCwGpJ.0kgZH2WXJgyJmrq0a3j63skib7WJYtKOXfsbHV8b9e + WxClOETCe03PtdD6G2sjEJSNFyTH_Qzzq6_21PO6kjmnEnBbibAnkiJbGhIJ + kOSqyp_vFqstpd38vtt7iLI8L3PkyZDQXS0hB1ZCOsZqBDGJXAoWFRBtxMSd + rMVkdvB6r8xJtn.1JrV1hpX4yRbCuEnCCPcwtGamlpyq5LG6YanKUVB868KF + UuZ4AHFwi.m_FYHalwtfCaArtWzYybl2nQQLjPbnXxqNvfwKt3ATKFEO40ZV + w1Ri7y.cO__09.eQHKIUNgMNeWgt.luD3thsEl0yz_ThzrCEkXDB1xAPNnLV + tb03RulEB0xNauYTuWgKR8WJzkO4LuXMlzNAAYBQLQy_t0GoezAs7Z4oq.CH + EfTK88cDJ7j7dXcXBi7q6g1NBZT3tyd9Bfn2DVdFaWAjWV9Lb8tir6J43MDP + byTrZ_zJxTWKgafhOxL0gZbd5xIEZ1eHHeQO5pVZlN6FR1awozFgS4NcZu5u + 5qRtn6zHo3zNe9ORwwxqlHAEJR_5I09WYSdmTxh2QkkDQLjSlwUNV4K8jxdH + L4ePIzNCQCt_bsGoG3uPXl8jtPD4sUWGY1lCeKAm.AHgZ.pSXXypMUpq4y14 + NihY89H61y5ZXo4Zd77shda_ +Received: from 176.119.200.160 (EHLO mail-200160.simplelogin.co) + by 10.217.151.74 with SMTPs + (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256); + Sun, 08 May 2022 11:12:34 +0000 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=simplelogin.co; + s=dkim; t=1652008349; h=From:To:Subject:Message-ID:Date; + bh=9HnrBUpZUe8OSXqTw1qF667IwLtHI8DqiyD0yAovIO4=; + b=PsxiMydvEQveb20xgUvvq3DhxlLyqqoPW7sC8d/pAm8tj7T2O+7z5xxR6vVbgz823Bglzc + djb3pRvNLgHnTozC+FiFOF8nVlWGybosn5oRfmNGkF9bhr0bJmfcDhiuC/tOaZKkod2lbf + jQ8bqMZhCsN/xVpkMqJdNJefdkj3dP4= +MIME-Version: 1.0 +Date: Sun, 8 May 2022 04:11:42 -0700 +Message-ID: + +Subject: MF +Content-Type: multipart/alternative; boundary="0000000000006dd95f05de7e2a70" +Content-Transfer-Encoding: 7bit +X-SimpleLogin-Type: Forward +X-SimpleLogin-EmailLog-ID: 41263490 +X-SimpleLogin-Envelope-From: {{ sender }} +X-SimpleLogin-Envelope-To: {{ rcpt }} +From: {{ sender }} +To: {{ rcpt_comma_list }} +List-Unsubscribe: +X-SimpleLogin-Want-Signing: yes +Content-Length: 473 + +--0000000000006dd95f05de7e2a70 +Content-Type: text/plain; charset="UTF-8" + +Here goes the original email content + +--0000000000006dd95f05de7e2a70-- + + +------=_Part_18136996_1734597748.1652016688604-- diff --git a/tests/handler/test_provider_complaints.py b/tests/handler/test_provider_complaints.py index 00b9a6a5..b222a752 100644 --- a/tests/handler/test_provider_complaints.py +++ b/tests/handler/test_provider_complaints.py @@ -1,51 +1,46 @@ -import email from email.message import Message -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText import pytest - from app.config import ( ALERT_COMPLAINT_FORWARD_PHASE, ALERT_COMPLAINT_REPLY_PHASE, ALERT_COMPLAINT_TRANSACTIONAL_PHASE, + POSTMASTER, ) from app.db import Session -from app.email import headers +from app.email import headers, status from app.handler.provider_complaint import ( handle_hotmail_complaint, handle_yahoo_complaint, ) from app.models import Alias, ProviderComplaint, SentAlert -from tests.utils import create_new_user +from tests.utils import create_new_user, load_eml_file origins = [ - [handle_yahoo_complaint, "yahoo", 6], - [handle_hotmail_complaint, "hotmail", 3], + [handle_yahoo_complaint, "yahoo"], + [handle_hotmail_complaint, "hotmail"], ] -def prepare_complaint(message: Message, part_num: int) -> Message: - complaint = MIMEMultipart("related") - # When walking, part 0 is the full message so we -1, and we want to be part N so -1 again - for i in range(part_num - 2): - document = MIMEText("text", "plain") - document.set_payload(f"Part {i}") - complaint.attach(document) - complaint.attach(message) - - return email.message_from_bytes(complaint.as_bytes()) +def prepare_complaint( + provider_name: str, rcpt_address: str, sender_address: str +) -> Message: + return load_eml_file( + f"{provider_name}_complaint.eml", + { + "postmaster": POSTMASTER, + "return_path": "sl.something.other@simplelogin.co", + "rcpt": rcpt_address, + "sender": sender_address, + "rcpt_comma_list": f"{rcpt_address},other_rcpt@somwhere.net", + }, + ) -@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_provider_to_user(flask_client, handle_ftor, provider, part_num): +@pytest.mark.parametrize("handle_ftor,provider", origins) +def test_provider_to_user(flask_client, handle_ftor, provider): user = create_new_user() - original_message = Message() - original_message[headers.TO] = user.email - original_message[headers.FROM] = "nobody@nowhere.net" - original_message.set_payload("Contents") - - complaint = prepare_complaint(original_message, part_num) + complaint = prepare_complaint(provider, user.email, "nobody@nowhere.net") assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 @@ -54,17 +49,12 @@ def test_provider_to_user(flask_client, handle_ftor, provider, part_num): assert alerts[0].alert_type == f"{ALERT_COMPLAINT_TRANSACTIONAL_PHASE}_{provider}" -@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_provider_forward_phase(flask_client, handle_ftor, provider, part_num): +@pytest.mark.parametrize("handle_ftor,provider", origins) +def test_provider_forward_phase(flask_client, handle_ftor, provider): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() - original_message = Message() - original_message[headers.TO] = "nobody@nowhere.net" - original_message[headers.FROM] = alias.email - original_message.set_payload("Contents") - - complaint = prepare_complaint(original_message, part_num) + complaint = prepare_complaint(provider, "nobody@nowhere.net", alias.email) assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 1 @@ -73,8 +63,8 @@ def test_provider_forward_phase(flask_client, handle_ftor, provider, part_num): assert alerts[0].alert_type == f"{ALERT_COMPLAINT_REPLY_PHASE}_{provider}" -@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_provider_reply_phase(flask_client, handle_ftor, provider, part_num): +@pytest.mark.parametrize("handle_ftor,provider", origins) +def test_provider_reply_phase(flask_client, handle_ftor, provider): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() @@ -83,7 +73,7 @@ def test_provider_reply_phase(flask_client, handle_ftor, provider, part_num): original_message[headers.FROM] = "no@no.no" original_message.set_payload("Contents") - complaint = prepare_complaint(original_message, part_num) + complaint = prepare_complaint(provider, alias.email, "no@no.no") assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 diff --git a/tests/test.env b/tests/test.env index 8189945f..a4ffe350 100644 --- a/tests/test.env +++ b/tests/test.env @@ -60,3 +60,5 @@ DMARC_CHECK_ENABLED=true PROTON_CLIENT_ID=to_fill PROTON_CLIENT_SECRET=to_fill PROTON_BASE_URL=https://localhost/api + +POSTMASTER=postmaster@simplelogin.co \ No newline at end of file From 6c13f7de05d6c8c2722579e12a8aaadf9fddb10c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Tue, 10 May 2022 18:23:14 +0200 Subject: [PATCH 2/9] refactored to reduce duplicated codepaths --- app/handler/provider_complaint.py | 67 ++++++++++++----------- tests/handler/test_provider_complaints.py | 2 +- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index c316fdba..92614809 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from io import BytesIO from mailbox import Message -from typing import Optional, List +from typing import Optional from app import s3 from app.config import ( @@ -13,13 +13,13 @@ from app.config import ( ) from app.email import headers from app.email_utils import ( - get_header_unicode, parse_full_address, save_email_for_debugging, to_bytes, render, send_email_with_rate_control, parse_address_list, + get_header_unicode, ) from app.log import LOG from app.models import ( @@ -44,13 +44,28 @@ class OriginalAddresses: class ProviderComplaintOrigin(ABC): @classmethod @abstractmethod - def get_original_message(cls, message: Message) -> Optional[Message]: + def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: pass @classmethod - @abstractmethod - def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: - pass + def sanitize_addresses( + cls, rcpt_header: Optional[str], message: Message + ) -> Optional[OriginalAddresses]: + try: + if not rcpt_header: + rcpt_header = message[headers.TO] + rcpt_list = parse_address_list(get_header_unicode(rcpt_header)) + if not rcpt_list: + saved_file = save_email_for_debugging(message, "NoRecipientComplaint") + LOG.w(f"Cannot find rcpt. Saved to {saved_file or 'nowhere'}") + return None + rcpt_address = rcpt_list[0][1] + _, sender_address = parse_full_address(message[headers.FROM]) + return OriginalAddresses(sender_address, rcpt_address) + except ValueError: + saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") + LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + return None @classmethod @abstractmethod @@ -73,6 +88,9 @@ class ProviderComplaintYahoo(ProviderComplaintOrigin): @classmethod def get_feedback_report(cls, message: Message) -> Optional[Message]: + """ + Find a report that yahoo embeds in the complaint. It has content type 'message/feedback-report' + """ for part in message.walk(): if part["content-type"] == "message/feedback-report": content = part.get_payload() @@ -83,20 +101,14 @@ class ProviderComplaintYahoo(ProviderComplaintOrigin): @classmethod def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + """ + Try to get the proper recipient from the report that yahoo adds as a port of the complaint. If we cannot find + the rcpt in the report or we can't find the report, use the first address in the original message from + """ report = cls.get_feedback_report(message) original = cls.get_original_message(message) - rcpt_address = report["original-rcpt-to"] - try: - if rcpt_address: - _, rcpt_address = parse_full_address(rcpt_address) - else: - rcpt_address = parse_address_list(original[headers.TO])[0] - _, sender_address = parse_full_address(original[headers.FROM]) - return OriginalAddresses(sender_address, rcpt_address) - except ValueError: - saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") - LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") - return False + rcpt_header = report["original-rcpt-to"] + return cls.sanitize_addresses(rcpt_header, original) @classmethod def name(cls): @@ -118,19 +130,12 @@ class ProviderComplaintHotmail(ProviderComplaintOrigin): @classmethod def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: - try: - part = cls.get_original_message(message) - rcpt_address = part["x-simplelogin-envelope-to"] - if rcpt_address: - _, rcpt_address = parse_full_address(rcpt_address) - else: - rcpt_address = parse_address_list(part[headers.TO])[0] - _, sender_address = parse_full_address(part[headers.FROM]) - return OriginalAddresses(sender_address, rcpt_address) - except ValueError: - saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") - LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") - return False + """ + Try to get the proper recipient from original x-simplelogin-envelope-to header we add on delivery. + If we can't find the header, use the first address in the original message from""" + original = cls.get_original_message(message) + rcpt_header = original["x-simplelogin-envelope-to"] + return cls.sanitize_addresses(rcpt_header, original) @classmethod def name(cls): diff --git a/tests/handler/test_provider_complaints.py b/tests/handler/test_provider_complaints.py index b222a752..5437343a 100644 --- a/tests/handler/test_provider_complaints.py +++ b/tests/handler/test_provider_complaints.py @@ -8,7 +8,7 @@ from app.config import ( POSTMASTER, ) from app.db import Session -from app.email import headers, status +from app.email import headers from app.handler.provider_complaint import ( handle_hotmail_complaint, handle_yahoo_complaint, From d2111d47681ff1c127ecc03dd1e09d5357243d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Tue, 10 May 2022 18:26:56 +0200 Subject: [PATCH 3/9] Added doc comments --- app/handler/provider_complaint.py | 5 +++++ pytest.ini | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index 92614809..a0711aa6 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -51,6 +51,11 @@ class ProviderComplaintOrigin(ABC): def sanitize_addresses( cls, rcpt_header: Optional[str], message: Message ) -> Optional[OriginalAddresses]: + """ + If the rcpt_header is not None, use it as the valid rcpt address, otherwise try to extract it from the To header + of the original message, since in the original message there can be more than one recipients. + There can only be one sender so that one can safely be extracted from the message headers. + """ try: if not rcpt_header: rcpt_header = message[headers.TO] diff --git a/pytest.ini b/pytest.ini index 3d362baf..c0f5472c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = +xaddopts = --cov --cov-config coverage.ini --cov-report=html:htmlcov From 48554369bd7f2118510a29962f49239cb1ca0767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Tue, 10 May 2022 23:34:57 +0200 Subject: [PATCH 4/9] Get the mailbox if possible from the email log --- app/email/headers.py | 1 + app/email_utils.py | 8 +- app/handler/provider_complaint.py | 99 +++++++++++++------ pytest.ini | 2 +- reset_local_db.sh | 7 ++ reset_test_db.sh | 6 ++ .../provider-complaint-forward-phase.html | 2 +- ...rovider-complaint-forward-phase.txt.jinja2 | 2 +- tests/handler/test_provider_complaints.py | 43 +++++--- 9 files changed, 122 insertions(+), 48 deletions(-) create mode 100755 reset_local_db.sh create mode 100755 reset_test_db.sh diff --git a/app/email/headers.py b/app/email/headers.py index 3edb4dce..5d176ff9 100644 --- a/app/email/headers.py +++ b/app/email/headers.py @@ -19,6 +19,7 @@ DKIM_SIGNATURE = "DKIM-Signature" X_SPAM_STATUS = "X-Spam-Status" LIST_UNSUBSCRIBE = "List-Unsubscribe" LIST_UNSUBSCRIBE_POST = "List-Unsubscribe-Post" +RETURN_PATH = "Return-Path" # headers used to DKIM sign in order of preference DKIM_HEADERS = [ diff --git a/app/email_utils.py b/app/email_utils.py index 47e46e2c..f52ee8e6 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1409,7 +1409,9 @@ def generate_verp_email( ).lower() -def get_verp_info_from_email(email: str) -> Optional[Tuple[VerpType, int]]: +def get_verp_info_from_email( + email: str, validate_time: bool = True +) -> Optional[Tuple[VerpType, int]]: """This method processes the email address, checks if it's a signed verp email generated by us to receive bounces and extracts the type of verp email and associated email log id/transactional email id stored as object_id """ @@ -1433,6 +1435,8 @@ def get_verp_info_from_email(email: str) -> Optional[Tuple[VerpType, int]]: # verp type, object_id, time if len(data) != 3: return None - if data[2] > (time.time() + VERP_MESSAGE_LIFETIME - VERP_TIME_START) / 60: + if validate_time and ( + data[2] > (time.time() + VERP_MESSAGE_LIFETIME - VERP_TIME_START) / 60 + ): return None return VerpType(data[0]), data[1] diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index a0711aa6..5ba2ec9f 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -20,6 +20,7 @@ from app.email_utils import ( send_email_with_rate_control, parse_address_list, get_header_unicode, + get_verp_info_from_email, ) from app.log import LOG from app.models import ( @@ -32,25 +33,44 @@ from app.models import ( Phase, ProviderComplaintState, RefusedEmail, + VerpType, + EmailLog, + Mailbox, ) @dataclass -class OriginalAddresses: - sender: str - recipient: str +class OriginalMessageInformation: + sender_address: str + rcpt_address: str + mailbox_address: Optional[str] class ProviderComplaintOrigin(ABC): @classmethod @abstractmethod - def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + def get_original_addresses( + cls, message: Message + ) -> Optional[OriginalMessageInformation]: pass @classmethod - def sanitize_addresses( + def _get_mailbox_id(cls, return_path: Optional[str]) -> Optional[Mailbox]: + if not return_path: + return None + _, return_path = parse_full_address(get_header_unicode(return_path)) + verp_type, email_log_id = get_verp_info_from_email(return_path) + if verp_type == VerpType.transactional: + return None + email_log = EmailLog.get_by(id=email_log_id) + if email_log: + return email_log.mailbox.email + return None + + @classmethod + def sanitize_addresses_and_extract_mailbox_id( cls, rcpt_header: Optional[str], message: Message - ) -> Optional[OriginalAddresses]: + ) -> Optional[OriginalMessageInformation]: """ If the rcpt_header is not None, use it as the valid rcpt address, otherwise try to extract it from the To header of the original message, since in the original message there can be more than one recipients. @@ -65,8 +85,15 @@ class ProviderComplaintOrigin(ABC): LOG.w(f"Cannot find rcpt. Saved to {saved_file or 'nowhere'}") return None rcpt_address = rcpt_list[0][1] - _, sender_address = parse_full_address(message[headers.FROM]) - return OriginalAddresses(sender_address, rcpt_address) + _, sender_address = parse_full_address( + get_header_unicode(message[headers.FROM]) + ) + + return OriginalMessageInformation( + sender_address, + rcpt_address, + cls._get_mailbox_id(message[headers.RETURN_PATH]), + ) except ValueError: saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") @@ -105,7 +132,9 @@ class ProviderComplaintYahoo(ProviderComplaintOrigin): return None @classmethod - def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + def get_original_addresses( + cls, message: Message + ) -> Optional[OriginalMessageInformation]: """ Try to get the proper recipient from the report that yahoo adds as a port of the complaint. If we cannot find the rcpt in the report or we can't find the report, use the first address in the original message from @@ -113,7 +142,7 @@ class ProviderComplaintYahoo(ProviderComplaintOrigin): report = cls.get_feedback_report(message) original = cls.get_original_message(message) rcpt_header = report["original-rcpt-to"] - return cls.sanitize_addresses(rcpt_header, original) + return cls.sanitize_addresses_and_extract_mailbox_id(rcpt_header, original) @classmethod def name(cls): @@ -134,13 +163,15 @@ class ProviderComplaintHotmail(ProviderComplaintOrigin): return None @classmethod - def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + def get_original_addresses( + cls, message: Message + ) -> Optional[OriginalMessageInformation]: """ Try to get the proper recipient from original x-simplelogin-envelope-to header we add on delivery. If we can't find the header, use the first address in the original message from""" original = cls.get_original_message(message) rcpt_header = original["x-simplelogin-envelope-to"] - return cls.sanitize_addresses(rcpt_header, original) + return cls.sanitize_addresses_and_extract_mailbox_id(rcpt_header, original) @classmethod def name(cls): @@ -164,50 +195,55 @@ def find_alias_with_address(address: str) -> Optional[Alias]: def handle_complaint(message: Message, origin: ProviderComplaintOrigin) -> bool: - addresses = origin.get_original_addresses(message) - if not addresses: + msg_info = origin.get_original_addresses(message) + if not msg_info: return False - user = User.get_by(email=addresses.recipient) + user = User.get_by(email=msg_info.rcpt_address) if user: LOG.d(f"Handle provider {origin.name()} complaint for {user}") - report_complaint_to_user_in_transactional_phase(user, origin) + report_complaint_to_user_in_transactional_phase(user, origin, msg_info) return True - alias = find_alias_with_address(addresses.sender) + alias = find_alias_with_address(msg_info.sender_address) # the email is during a reply phase, from=alias and to=destination if alias: LOG.i( - f"Complaint from {origin.name} during reply phase {alias} -> {addresses.recipient}, {user}" + f"Complaint from {origin.name} during reply phase {alias} -> {msg_info.rcpt_address}, {user}" + ) + report_complaint_to_user_in_reply_phase( + alias, msg_info.rcpt_address, origin, msg_info ) - report_complaint_to_user_in_reply_phase(alias, addresses.recipient, origin) store_provider_complaint(alias, message) return True - contact = Contact.get_by(reply_email=addresses.sender) + contact = Contact.get_by(reply_email=msg_info.sender_address) if contact: alias = contact.alias else: - alias = find_alias_with_address(addresses.recipient) + alias = find_alias_with_address(msg_info.rcpt_address) if not alias: LOG.e( - f"Cannot find alias for address {addresses.recipient} or contact with reply {addresses.sender}" + f"Cannot find alias for address {msg_info.rcpt_address} or contact with reply {msg_info.sender_address}" ) return False - report_complaint_to_user_in_forward_phase(alias, origin) + report_complaint_to_user_in_forward_phase(alias, origin, msg_info) return True def report_complaint_to_user_in_reply_phase( - alias: Alias, to_address: str, origin: ProviderComplaintOrigin + alias: Alias, + to_address: str, + origin: ProviderComplaintOrigin, + msg_info: OriginalMessageInformation, ): capitalized_name = origin.name().capitalize() send_email_with_rate_control( alias.user, f"{ALERT_COMPLAINT_REPLY_PHASE}_{origin.name()}", - alias.user.email, + msg_info.mailbox_address or alias.mailbox.email, f"Abuse report from {capitalized_name}", render( "transactional/provider-complaint-reply-phase.txt.jinja2", @@ -222,13 +258,13 @@ def report_complaint_to_user_in_reply_phase( def report_complaint_to_user_in_transactional_phase( - user: User, origin: ProviderComplaintOrigin + user: User, origin: ProviderComplaintOrigin, msg_info: OriginalMessageInformation ): capitalized_name = origin.name().capitalize() send_email_with_rate_control( user, f"{ALERT_COMPLAINT_TRANSACTIONAL_PHASE}_{origin.name()}", - user.email, + msg_info.mailbox_address or user.email, f"Abuse report from {capitalized_name}", render( "transactional/provider-complaint-to-user.txt.jinja2", @@ -246,23 +282,24 @@ def report_complaint_to_user_in_transactional_phase( def report_complaint_to_user_in_forward_phase( - alias: Alias, origin: ProviderComplaintOrigin + alias: Alias, origin: ProviderComplaintOrigin, msg_info: OriginalMessageInformation ): capitalized_name = origin.name().capitalize() user = alias.user + mailbox_email = msg_info.mailbox_address or alias.mailbox.email send_email_with_rate_control( user, f"{ALERT_COMPLAINT_FORWARD_PHASE}_{origin.name()}", - user.email, + mailbox_email, f"Abuse report from {capitalized_name}", render( "transactional/provider-complaint-forward-phase.txt.jinja2", - user=user, + email=mailbox_email, provider=capitalized_name, ), render( "transactional/provider-complaint-forward-phase.html", - user=user, + email=mailbox_email, provider=capitalized_name, ), max_nb_alert=1, diff --git a/pytest.ini b/pytest.ini index c0f5472c..3d362baf 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -xaddopts = +addopts = --cov --cov-config coverage.ini --cov-report=html:htmlcov diff --git a/reset_local_db.sh b/reset_local_db.sh new file mode 100755 index 00000000..f194f335 --- /dev/null +++ b/reset_local_db.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +export DB_URI=postgresql://myuser:mypassword@localhost:15432/simplelogin +echo 'drop schema public cascade; create schema public;' | psql $DB_URI + +poetry run alembic upgrade head +poetry run flask dummy-data diff --git a/reset_test_db.sh b/reset_test_db.sh new file mode 100755 index 00000000..ce392919 --- /dev/null +++ b/reset_test_db.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +export DB_URI=postgresql://myuser:mypassword@localhost:15432/test +echo 'drop schema public cascade; create schema public;' | psql $DB_URI + +poetry run alembic upgrade head diff --git a/templates/emails/transactional/provider-complaint-forward-phase.html b/templates/emails/transactional/provider-complaint-forward-phase.html index a1502b1f..cefeedaf 100644 --- a/templates/emails/transactional/provider-complaint-forward-phase.html +++ b/templates/emails/transactional/provider-complaint-forward-phase.html @@ -6,7 +6,7 @@ {% endcall %} {% call text() %} - {{ provider }} has informed us about an email sent to {{ user.email }} that might have been considered as spam, + {{ provider }} has informed us about an email sent to {{ email }} that might have been considered as spam, either by you or by {{ provider }} spam filter. {% endcall %} diff --git a/templates/emails/transactional/provider-complaint-forward-phase.txt.jinja2 b/templates/emails/transactional/provider-complaint-forward-phase.txt.jinja2 index 428674e1..b76c6714 100644 --- a/templates/emails/transactional/provider-complaint-forward-phase.txt.jinja2 +++ b/templates/emails/transactional/provider-complaint-forward-phase.txt.jinja2 @@ -5,7 +5,7 @@ Hi, This is SimpleLogin team. -{{ provider }} has informed us about an email sent to {{ user.email }} that might have been considered as spam, +{{ provider }} has informed us about an email sent to {{ email }} that might have been considered as spam, either by you or by {{ provider }}. Please note that explicitly marking a SimpleLogin's forwarded email as Spam diff --git a/tests/handler/test_provider_complaints.py b/tests/handler/test_provider_complaints.py index 5437343a..8452cd71 100644 --- a/tests/handler/test_provider_complaints.py +++ b/tests/handler/test_provider_complaints.py @@ -8,12 +8,19 @@ from app.config import ( POSTMASTER, ) from app.db import Session -from app.email import headers +from app.email_utils import generate_verp_email from app.handler.provider_complaint import ( handle_hotmail_complaint, handle_yahoo_complaint, ) -from app.models import Alias, ProviderComplaint, SentAlert +from app.models import ( + Alias, + ProviderComplaint, + SentAlert, + EmailLog, + VerpType, + Contact, +) from tests.utils import create_new_user, load_eml_file origins = [ @@ -23,13 +30,28 @@ origins = [ def prepare_complaint( - provider_name: str, rcpt_address: str, sender_address: str + provider_name: str, alias: Alias, rcpt_address: str, sender_address: str ) -> Message: + contact = Contact.create( + user_id=alias.user.id, + alias_id=alias.id, + website_email="a@b.c", + reply_email="d@e.f", + commit=True, + ) + elog = EmailLog.create( + user_id=alias.user.id, + mailbox_id=alias.user.default_mailbox_id, + contact_id=contact.id, + commit=True, + bounced=True, + ) + return_path = generate_verp_email(VerpType.bounce_forward, elog.id) return load_eml_file( f"{provider_name}_complaint.eml", { "postmaster": POSTMASTER, - "return_path": "sl.something.other@simplelogin.co", + "return_path": return_path, "rcpt": rcpt_address, "sender": sender_address, "rcpt_comma_list": f"{rcpt_address},other_rcpt@somwhere.net", @@ -40,7 +62,9 @@ def prepare_complaint( @pytest.mark.parametrize("handle_ftor,provider", origins) def test_provider_to_user(flask_client, handle_ftor, provider): user = create_new_user() - complaint = prepare_complaint(provider, user.email, "nobody@nowhere.net") + alias = Alias.create_new_random(user) + Session.commit() + complaint = prepare_complaint(provider, alias, user.email, "nobody@nowhere.net") assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 @@ -54,7 +78,7 @@ def test_provider_forward_phase(flask_client, handle_ftor, provider): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() - complaint = prepare_complaint(provider, "nobody@nowhere.net", alias.email) + complaint = prepare_complaint(provider, alias, "nobody@nowhere.net", alias.email) assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 1 @@ -68,12 +92,7 @@ def test_provider_reply_phase(flask_client, handle_ftor, provider): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() - original_message = Message() - original_message[headers.TO] = alias.email - original_message[headers.FROM] = "no@no.no" - original_message.set_payload("Contents") - - complaint = prepare_complaint(provider, alias.email, "no@no.no") + complaint = prepare_complaint(provider, alias, alias.email, "no@no.no") assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 From bf577a60219546b3921d1e2de8f1f5802b2ff00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 12 May 2022 17:05:28 +0200 Subject: [PATCH 5/9] Move scripts to scripts dir --- reset_local_db.sh => scripts/reset_local_db.sh | 0 reset_test_db.sh => scripts/reset_test_db.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename reset_local_db.sh => scripts/reset_local_db.sh (100%) rename reset_test_db.sh => scripts/reset_test_db.sh (100%) diff --git a/reset_local_db.sh b/scripts/reset_local_db.sh similarity index 100% rename from reset_local_db.sh rename to scripts/reset_local_db.sh diff --git a/reset_test_db.sh b/scripts/reset_test_db.sh similarity index 100% rename from reset_test_db.sh rename to scripts/reset_test_db.sh From 3e0cb546a2c1793058531f8f8a8e5ba2ae0d0501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 13 May 2022 14:42:20 +0200 Subject: [PATCH 6/9] Added docs --- CONTRIBUTING.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d558e3e3..ebd12459 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,6 +105,15 @@ We cannot use the local database to generate migration script as the local datab It is created via `db.create_all()` (cf `fake_data()` method). This is convenient for development and unit tests as we don't have to wait for the migration. +## Reset database + +There are two scripts to reset your local db to an empty state: + +- `scripts/reset_local_db.sh` will reset your development db to the latest migration version and add the development data needed to run the +server.py locally. +- `scripts/reset_test_db.sh` will reset your test db to the latest migration without adding the dev server data to prevent interferring with +the tests. + ## Code structure The repo consists of the three following entry points: From 64c67f44294fb18078bfc87e6a2d693a637619ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 13 May 2022 18:14:21 +0200 Subject: [PATCH 7/9] PR comments --- app/email_utils.py | 22 ++++++++++++---------- tests/test.env | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index f52ee8e6..027d16c9 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1,4 +1,5 @@ import base64 +import binascii import enum import hmac import json @@ -1409,9 +1410,7 @@ def generate_verp_email( ).lower() -def get_verp_info_from_email( - email: str, validate_time: bool = True -) -> Optional[Tuple[VerpType, int]]: +def get_verp_info_from_email(email: str) -> Optional[Tuple[VerpType, int]]: """This method processes the email address, checks if it's a signed verp email generated by us to receive bounces and extracts the type of verp email and associated email log id/transactional email id stored as object_id """ @@ -1422,10 +1421,15 @@ def get_verp_info_from_email( fields = username.split(".") if len(fields) != 3 or fields[0] != VERP_PREFIX: return None - padding = (8 - (len(fields[1]) % 8)) % 8 - payload = base64.b32decode(fields[1].encode("utf-8").upper() + (b"=" * padding)) - padding = (8 - (len(fields[2]) % 8)) % 8 - signature = base64.b32decode(fields[2].encode("utf-8").upper() + (b"=" * padding)) + try: + padding = (8 - (len(fields[1]) % 8)) % 8 + payload = base64.b32decode(fields[1].encode("utf-8").upper() + (b"=" * padding)) + padding = (8 - (len(fields[2]) % 8)) % 8 + signature = base64.b32decode( + fields[2].encode("utf-8").upper() + (b"=" * padding) + ) + except binascii.Error: + return None expected_signature = hmac.new( VERP_EMAIL_SECRET.encode("utf-8"), payload, VERP_HMAC_ALGO ).digest()[:8] @@ -1435,8 +1439,6 @@ def get_verp_info_from_email( # verp type, object_id, time if len(data) != 3: return None - if validate_time and ( - data[2] > (time.time() + VERP_MESSAGE_LIFETIME - VERP_TIME_START) / 60 - ): + if data[2] > (time.time() + VERP_MESSAGE_LIFETIME - VERP_TIME_START) / 60: return None return VerpType(data[0]), data[1] diff --git a/tests/test.env b/tests/test.env index a4ffe350..84464e39 100644 --- a/tests/test.env +++ b/tests/test.env @@ -61,4 +61,4 @@ PROTON_CLIENT_ID=to_fill PROTON_CLIENT_SECRET=to_fill PROTON_BASE_URL=https://localhost/api -POSTMASTER=postmaster@simplelogin.co \ No newline at end of file +POSTMASTER=postmaster@test.domain \ No newline at end of file From 3578c613667d5d7db5bffdff7451a187b5f669b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 13 May 2022 19:18:20 +0200 Subject: [PATCH 8/9] Use header --- app/handler/provider_complaint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index 5ba2ec9f..9b9f5cdf 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -170,7 +170,7 @@ class ProviderComplaintHotmail(ProviderComplaintOrigin): Try to get the proper recipient from original x-simplelogin-envelope-to header we add on delivery. If we can't find the header, use the first address in the original message from""" original = cls.get_original_message(message) - rcpt_header = original["x-simplelogin-envelope-to"] + rcpt_header = original[headers.SL_ENVELOPE_TO] return cls.sanitize_addresses_and_extract_mailbox_id(rcpt_header, original) @classmethod From 0da2fd94f18a7236972cc3c00d1af78c90013c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Mon, 16 May 2022 10:16:42 +0200 Subject: [PATCH 9/9] Set header as a constant --- app/email/headers.py | 3 +++ app/handler/provider_complaint.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/email/headers.py b/app/email/headers.py index 5d176ff9..800a5a2b 100644 --- a/app/email/headers.py +++ b/app/email/headers.py @@ -51,3 +51,6 @@ MIME_HEADERS = [h.lower() for h in MIME_HEADERS] # according to https://datatracker.ietf.org/doc/html/rfc3834#section-3.1.7, this header should be set to "auto-replied" # however on hotmail, this is set to "auto-generated" AUTO_SUBMITTED = "Auto-Submitted" + +# Yahoo complaint specific header +YAHOO_ORIGINAL_RECIPIENT = "original-rcpt-to" diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index 9b9f5cdf..ef7453b6 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -141,7 +141,7 @@ class ProviderComplaintYahoo(ProviderComplaintOrigin): """ report = cls.get_feedback_report(message) original = cls.get_original_message(message) - rcpt_header = report["original-rcpt-to"] + rcpt_header = report[headers.YAHOO_ORIGINAL_RECIPIENT] return cls.sanitize_addresses_and_extract_mailbox_id(rcpt_header, original) @classmethod