diff --git a/app/email_utils.py b/app/email_utils.py index a899ad5f..47e46e2c 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1314,6 +1314,20 @@ def should_ignore_bounce(mail_from: str) -> bool: return False +def parse_address_list(address_list: str) -> List[Tuple[str, str]]: + """ + Parse a list of email addresses from a header in the form "ab , cd " + and return a list [("ab", "ab@sd.com"),("cd", "cd@cd.com")] + """ + processed_addresses = [] + for split_address in address_list.split(","): + split_address = split_address.strip() + if not split_address: + continue + processed_addresses.append(parse_full_address(split_address)) + return processed_addresses + + def parse_full_address(full_address) -> (str, str): """ parse the email address full format and return the display name and address diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index ce2befec..c316fdba 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -1,8 +1,9 @@ import uuid from abc import ABC, abstractmethod +from dataclasses import dataclass from io import BytesIO from mailbox import Message -from typing import Optional +from typing import Optional, List from app import s3 from app.config import ( @@ -18,6 +19,7 @@ from app.email_utils import ( to_bytes, render, send_email_with_rate_control, + parse_address_list, ) from app.log import LOG from app.models import ( @@ -33,12 +35,23 @@ from app.models import ( ) +@dataclass +class OriginalAddresses: + sender: str + recipient: str + + class ProviderComplaintOrigin(ABC): @classmethod @abstractmethod def get_original_message(cls, message: Message) -> Optional[Message]: pass + @classmethod + @abstractmethod + def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + pass + @classmethod @abstractmethod def name(cls): @@ -58,6 +71,33 @@ class ProviderComplaintYahoo(ProviderComplaintOrigin): return part return None + @classmethod + def get_feedback_report(cls, message: Message) -> Optional[Message]: + for part in message.walk(): + if part["content-type"] == "message/feedback-report": + content = part.get_payload() + if not content: + continue + return content[0] + return None + + @classmethod + def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + report = cls.get_feedback_report(message) + original = cls.get_original_message(message) + rcpt_address = report["original-rcpt-to"] + try: + if rcpt_address: + _, rcpt_address = parse_full_address(rcpt_address) + else: + rcpt_address = parse_address_list(original[headers.TO])[0] + _, sender_address = parse_full_address(original[headers.FROM]) + return OriginalAddresses(sender_address, rcpt_address) + except ValueError: + saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") + LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + return False + @classmethod def name(cls): return "yahoo" @@ -76,6 +116,22 @@ class ProviderComplaintHotmail(ProviderComplaintOrigin): return part return None + @classmethod + def get_original_addresses(cls, message: Message) -> Optional[OriginalAddresses]: + try: + part = cls.get_original_message(message) + rcpt_address = part["x-simplelogin-envelope-to"] + if rcpt_address: + _, rcpt_address = parse_full_address(rcpt_address) + else: + rcpt_address = parse_address_list(part[headers.TO])[0] + _, sender_address = parse_full_address(part[headers.FROM]) + return OriginalAddresses(sender_address, rcpt_address) + except ValueError: + saved_file = save_email_for_debugging(message, "ComplaintOriginalAddress") + LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + return False + @classmethod def name(cls): return "hotmail" @@ -98,45 +154,35 @@ def find_alias_with_address(address: str) -> Optional[Alias]: def handle_complaint(message: Message, origin: ProviderComplaintOrigin) -> bool: - original_message = origin.get_original_message(message) - - try: - _, to_address = parse_full_address( - get_header_unicode(original_message[headers.TO]) - ) - _, from_address = parse_full_address( - get_header_unicode(original_message[headers.FROM]) - ) - except ValueError: - saved_file = save_email_for_debugging(message, "FromParseFailed") - LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + addresses = origin.get_original_addresses(message) + if not addresses: return False - user = User.get_by(email=to_address) + user = User.get_by(email=addresses.recipient) if user: LOG.d(f"Handle provider {origin.name()} complaint for {user}") report_complaint_to_user_in_transactional_phase(user, origin) return True - alias = find_alias_with_address(from_address) + alias = find_alias_with_address(addresses.sender) # the email is during a reply phase, from=alias and to=destination if alias: LOG.i( - f"Complaint from {origin.name} during reply phase {alias} -> {to_address}, {user}" + f"Complaint from {origin.name} during reply phase {alias} -> {addresses.recipient}, {user}" ) - report_complaint_to_user_in_reply_phase(alias, to_address, origin) + report_complaint_to_user_in_reply_phase(alias, addresses.recipient, origin) store_provider_complaint(alias, message) return True - contact = Contact.get_by(reply_email=from_address) + contact = Contact.get_by(reply_email=addresses.sender) if contact: alias = contact.alias else: - alias = find_alias_with_address(to_address) + alias = find_alias_with_address(addresses.recipient) if not alias: LOG.e( - f"Cannot find alias from address {to_address} or contact with reply {from_address}" + f"Cannot find alias for address {addresses.recipient} or contact with reply {addresses.sender}" ) return False diff --git a/tests/example_emls/hotmail_complaint.eml b/tests/example_emls/hotmail_complaint.eml new file mode 100644 index 00000000..a3fbeda7 --- /dev/null +++ b/tests/example_emls/hotmail_complaint.eml @@ -0,0 +1,258 @@ +X-SimpleLogin-Client-IP: 40.92.66.13 +Received-SPF: Pass (mailfrom) identity=mailfrom; client-ip=40.92.66.13; + helo=eur01-ve1-obe.outbound.protection.outlook.com; + envelope-from=staff@hotmail.com; receiver= +Received: from EUR01-VE1-obe.outbound.protection.outlook.com + (mail-oln040092066013.outbound.protection.outlook.com [40.92.66.13]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by prod4.simplelogin.co (Postfix) with ESMTPS id 408E09C472 + for <{{ postmaster }}>; Mon, 9 May 2022 13:11:34 +0000 (UTC) +ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none; + b=V3N8KdYGgYrjs5KcjFUA0MgPUmOc+NV4ygLfSd7fehfiNemKdhe6Cpfj58zWFNzoG5qBoUCIm/BI7aCr7lqAU2hQJypTrJG+3zbSdnuCKMBVV5GHZxkE+XAeSU+4wt4xwl1ZiVx/2P//xUVWN/TVmiuKUgCn9n+WagU9LYGVT9z6wwOpXggpDf6ow9RnJDPJpkakHRh7rQPABbrOpVqEZnoJdAH5mgdTHJOeBumNym4i3GKnky+IfMlqwGcbTrzgrt/D3PpZdsMG4B+jEHtTo3FgB9JY+abjU9Bvn4rXwKr3RMF+1ZV3UsznQVwuT99PtfEcExV3zSsqEPDBy9QT9w== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; + s=arcselector9901; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1; + bh=Y37p6EaXY5hpBNgMr1ILYzy35GKdkqWXm69FR2RyQgA=; + b=aet1P4fpmUM9bqbLD3vtp/EWfUi2WfvWbOnnLg/YZ2vxoTF/eM5IHDBB/I7btdzZICric+KkhRih/kvaVURGy4jybYjn9FNfT+HShTJa75Pk30fp3in/5lL2x6Q0xM0Naf9YtTvGgqlLDrdgCmktxyByNAOFPo27fEWy3fk/00IPWyI8j77VvYsGn8rJCLbhDUBWwGzQ9P7SabIqn9Ybx6CKcw2FssJhSNAyOIx7EkrGxq8y/5dXeWSHLFBdHPu6F9w/DKyt9cv17rBSnHo4tx1Ese93vBHT5XIwTwnGisCa0++eqL/69GugKoe5odkAfsdRAlBjVTgXp2Lol4rrpg== +ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none; + dkim=none; arc=none +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=hotmail.com; + s=selector1; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=Y37p6EaXY5hpBNgMr1ILYzy35GKdkqWXm69FR2RyQgA=; + b=uMkd90Lx6ikNpk7RRBU3AfQ0jjbjRZAGQLnY3r+dQ3CNnhgfHxpNRudxGDydmf6GQ2AuylmOnLVATh8XMKTvCnVg8hjB9xrxd5qPpQ3k92U5VlgVe1o1Nwq8R6VCJugOZduDjSJdBXO2ACosUul6IQXKMBpSNq+bGJ9VHu63EGTphkWOOw1a4PArg8tQTSmkpkyh788nsfNXnVsh2fkL6we1LyvagQzTS4e1ynuSk1zAk+6U5KOuhRVr2Nh/AvyvswWpjA4pflOqFwyqsMYb3N6wnpRTct8CJUPlQwEx6chiJgKNGrAkdRbnWaEyeIEdyJB/NLwtPqZzKYFgv7f8wg== +Received: from AM6PR02CA0021.eurprd02.prod.outlook.com (2603:10a6:20b:6e::34) + by AM0PR02MB4563.eurprd02.prod.outlook.com (2603:10a6:208:ec::33) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.22; Mon, 9 May + 2022 13:11:32 +0000 +Received: from AM6EUR05FT047.eop-eur05.prod.protection.outlook.com + (2603:10a6:20b:6e:cafe::26) by AM6PR02CA0021.outlook.office365.com + (2603:10a6:20b:6e::34) with Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.23 via Frontend + Transport; Mon, 9 May 2022 13:11:32 +0000 +Received: from DM5SVC01SF077 (40.107.211.126) by + AM6EUR05FT047.mail.protection.outlook.com (10.233.241.167) with Microsoft + SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id + 15.20.5227.15 via Frontend Transport; Mon, 9 May 2022 13:11:32 +0000 +X-IncomingTopHeaderMarker: + OriginalChecksum:86053024C4DD515561A96BAF61AACB6F8A4DB30C8D14CAC5F2F7D189ACDCA109;UpperCasedChecksum:5323AB267D58619B82076460438A30DFDD8E7969870D76B723156F921928319B;SizeAsReceived:257;Count:6 +Date: Mon, 9 May 2022 13:10:08 +0000 +From: +Subject: complaint about message from 176.119.200.162 +To: {{ postmaster }} +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="31A9507D-D0B3-4DCD-AFBB-413468892CFE" +X-IncomingHeaderCount: 6 +Message-ID: + <1d63d9ee-8f3e-4876-955c-1807db5ad138@AM6EUR05FT047.eop-eur05.prod.protection.outlook.com> +X-EOPAttributedMessage: 0 +X-MS-PublicTrafficType: Email +X-MS-Office365-Filtering-Correlation-Id: 44e9ec0b-6c5d-4cea-6417-08da31bd7000 +X-MS-TrafficTypeDiagnostic: AM0PR02MB4563:EE_ +X-Microsoft-Antispam: BCL:0; +X-Microsoft-Antispam-Message-Info: + lK5xD4UZS47NfR0tHc3wEp4HHOifZ4SDBb8aKx7H/vEW8Rg8rXXH12G4lWdpzr8qTsCmvzuhj5x6IAumOKQ8lWLj5Lp3jyml91wVnwCtUnk5cTXpQwDZd9QMgtEW07GoLdWjkbShAhLRDf+9Y4DxidHCacOAYxcNX42wo3vYZOEHDzVRUxSmY0c7Km60pDtiYzEk+P9AoE2YKYG2rDwDx0vgoLgqFspGqQ+2OeHD2ZAEyATHR/sQy6tf5S2d4wA3HcHrwrGMlz/4d9VbT5h9a5cqj9S59wpuc6g8nyYhmK3AHJkB5nXmpBZBihTw5X/Qh5PZqUYwPxkwpq3WlaEuXvzaKFiwJFvtuRGX+mEioClCxiwPROb7sI9ZHWPw48AHysF+whYGBfleRy4c2SuW6e1D5uewGry+lXVljxg7qKo= +X-OriginatorOrg: sct-15-20-4755-11-msonline-outlook-ab7de.templateTenant +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 09 May 2022 13:11:32.0875 + (UTC) +X-MS-Exchange-CrossTenant-Network-Message-Id: + 44e9ec0b-6c5d-4cea-6417-08da31bd7000 +X-MS-Exchange-CrossTenant-Id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa +X-MS-Exchange-CrossTenant-AuthSource: + AM6EUR05FT047.eop-eur05.prod.protection.outlook.com +X-MS-Exchange-CrossTenant-AuthAs: Anonymous +X-MS-Exchange-CrossTenant-FromEntityHeader: Internet +X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: + 00000000-0000-0000-0000-000000000000 +X-MS-Exchange-Transport-CrossTenantHeadersStamped: AM0PR02MB4563 +X-Spamd-Result: default: False [-1.75 / 13.00]; + ARC_ALLOW(-1.00)[microsoft.com:s=arcselector9901:i=1]; + DMARC_POLICY_ALLOW(-0.50)[hotmail.com,none]; + R_SPF_ALLOW(-0.20)[+ip4:40.92.0.0/15]; + MIME_HTML_ONLY(0.20)[]; + R_DKIM_ALLOW(-0.20)[hotmail.com:s=selector1]; + MIME_GOOD(-0.10)[multipart/mixed,multipart/related]; + MANY_INVISIBLE_PARTS(0.05)[1]; + NEURAL_HAM(-0.00)[-0.996]; + FROM_EQ_ENVFROM(0.00)[]; + FREEMAIL_ENVFROM(0.00)[hotmail.com]; + MIME_TRACE(0.00)[0:+,1:~,2:+,3:+,4:~]; + ASN(0.00)[asn:8075, ipnet:40.80.0.0/12, country:US]; + RCVD_IN_DNSWL_NONE(0.00)[40.92.66.13:from]; + DKIM_TRACE(0.00)[hotmail.com:+]; + RCVD_TLS_LAST(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + FREEMAIL_FROM(0.00)[hotmail.com]; + FROM_NO_DN(0.00)[]; + TO_DN_NONE(0.00)[]; + RCVD_COUNT_THREE(0.00)[4]; + RCPT_COUNT_ONE(0.00)[1]; + DWL_DNSWL_NONE(0.00)[hotmail.com:dkim] +X-Rspamd-Queue-Id: 408E09C472 +X-Rspamd-Server: prod4 +Content-Transfer-Encoding: 7bit + +--31A9507D-D0B3-4DCD-AFBB-413468892CFE +Content-Type: message/rfc822 +Content-Disposition: inline + +X-HmXmrOriginalRecipient: +X-MS-Exchange-EOPDirect: true +Received: from SJ0PR11MB4958.namprd11.prod.outlook.com (2603:10b6:a03:2ae::24) + by SA0PR11MB4525.namprd11.prod.outlook.com with HTTPS; Mon, 9 May 2022 + 04:30:48 +0000 +Received: from BN9PR03CA0117.namprd03.prod.outlook.com (2603:10b6:408:fd::32) + by SJ0PR11MB4958.namprd11.prod.outlook.com (2603:10b6:a03:2ae::24) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.20; Mon, 9 May + 2022 04:30:45 +0000 +Received: from BN8NAM11FT053.eop-nam11.prod.protection.outlook.com + (2603:10b6:408:fd:cafe::d0) by BN9PR03CA0117.outlook.office365.com + (2603:10b6:408:fd::32) with Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5227.20 via Frontend + Transport; Mon, 9 May 2022 04:30:45 +0000 +Authentication-Results: spf=pass (sender IP is 176.119.200.162) + smtp.mailfrom=simplelogin.co; dkim=pass (signature was verified) + header.d=simplelogin.co;dmarc=pass action=none + header.from=simplelogin.co;compauth=pass reason=100 +Received-SPF: Pass (protection.outlook.com: domain of simplelogin.co + designates 176.119.200.162 as permitted sender) + receiver=protection.outlook.com; client-ip=176.119.200.162; + helo=mail-200162.simplelogin.co; +Received: from mail-200162.simplelogin.co (176.119.200.162) by + BN8NAM11FT053.mail.protection.outlook.com (10.13.177.209) with Microsoft SMTP + Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id + 15.20.5227.15 via Frontend Transport; Mon, 9 May 2022 04:30:44 +0000 +X-IncomingTopHeaderMarker: + OriginalChecksum:5EBD8C309CA888838EDC898C63E28E1EC00EF74772276A54C08DA83D658756F4;UpperCasedChecksum:E102374CD208D4ACB2034F1A17F76DA6345BD176395C6D4EADEC3B47BFF41ECC;SizeAsReceived:1262;Count:15 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=simplelogin.co; + s=dkim; t=1652070640; h=From:To:Subject:Message-ID:Date; + bh=Tu2Q0oO5GuGw4CVxDAdixtRKr6jqMWjpH9zEf50uKwg=; + b=o6I0Ij1CahU9EUj/9uwWJpsDjfi/2gQIXT0KJT6IAK9hOoJ5bVqPsqtyGTfIoqYhhtD/ic + 5NybKJmB6B6KL5hl5LG3KzCdaWfe3dAAhD4e2gIU80dal596dlzluyvLR1k+6rdM4JvlGq + OVWLR42Oj4anrnOqLCUkL44ILIhLpAE= +Date: Mon, 9 May 2022 00:30:38 -0400 (EDT) +Message-ID: + <10627474.1041327707.1652070638478.JavaMail.cloud@p2-mta-0301.p2.messagegears.net> +Subject: Original Subject +Content-Type: multipart/mixed; + boundary="----=_Part_1041327705_575167926.1652070638478" +Content-Transfer-Encoding: 7bit +X-SimpleLogin-Type: Forward +X-SimpleLogin-EmailLog-ID: 832832 +X-SimpleLogin-Envelope-To: {{ rcpt }} +From: {{ sender }} +Reply-To: {{ sender }} +To: {{ rcpt_comma_list }} +List-Unsubscribe: +X-SimpleLogin-Want-Signing: yes +X-IncomingHeaderCount: 15 +Return-Path: {{ return_path }} +X-MS-Exchange-Organization-ExpirationStartTime: 09 May 2022 04:30:45.1195 + (UTC) +X-MS-Exchange-Organization-ExpirationStartTimeReason: OriginalSubmit +X-MS-Exchange-Organization-ExpirationInterval: 1:00:00:00.0000000 +X-MS-Exchange-Organization-ExpirationIntervalReason: OriginalSubmit +X-MS-Exchange-Organization-Network-Message-Id: + ede92e41-5acb-4474-c5be-08da3174af2b +X-EOPAttributedMessage: 0 +X-EOPTenantAttributedMessage: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa:0 +X-MS-Exchange-Organization-MessageDirectionality: Incoming +X-MS-PublicTrafficType: Email +X-MS-Exchange-Organization-AuthSource: + BN8NAM11FT053.eop-nam11.prod.protection.outlook.com +X-MS-Exchange-Organization-AuthAs: Anonymous +X-MS-UserLastLogonTime: 5/9/2022 3:30:52 AM +X-MS-Office365-Filtering-Correlation-Id: ede92e41-5acb-4474-c5be-08da3174af2b +X-MS-TrafficTypeDiagnostic: SJ0PR11MB4958:EE_ +X-MS-Exchange-EOPDirect: true +X-Sender-IP: 176.119.200.162 +X-SID-PRA: PHWNQHFTTLQNZJXKMLHZCSKLLLJXMGEJOEOWW@SIMPLELOGIN.CO +X-SID-Result: PASS +X-MS-Exchange-Organization-PCL: 2 +X-MS-Exchange-Organization-SCL: 1 +X-Microsoft-Antispam: BCL:0; +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 09 May 2022 04:30:44.9945 + (UTC) +X-MS-Exchange-CrossTenant-Network-Message-Id: + ede92e41-5acb-4474-c5be-08da3174af2b +X-MS-Exchange-CrossTenant-Id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa +X-MS-Exchange-CrossTenant-AuthSource: + BN8NAM11FT053.eop-nam11.prod.protection.outlook.com +X-MS-Exchange-CrossTenant-AuthAs: Anonymous +X-MS-Exchange-CrossTenant-FromEntityHeader: Internet +X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: + 00000000-0000-0000-0000-000000000000 +X-MS-Exchange-Transport-CrossTenantHeadersStamped: SJ0PR11MB4958 +X-MS-Exchange-Transport-EndToEndLatency: 00:00:03.3271765 +X-MS-Exchange-Processed-By-BccFoldering: 15.20.5227.023 +X-Microsoft-Antispam-Mailbox-Delivery: + abwl:0;wl:0;pcwl:0;kl:0;iwl:0;ijl:0;dwl:0;dkl:0;rwl:0;ucf:0;jmr:0;ex:0;auth:1;dest:I;ENG:(5062000285)(90000117)(90005022)(91005020)(91035115)(5061607266)(5061608174)(9050020)(9100338)(2008001134)(2008000189)(2008120399)(2008019284)(2008021020)(8390246)(8377080)(8386120)(4810004)(4910013)(9910022)(9510006)(10110021)(9320005); +X-Message-Info: + 5vMbyqxGkdcvoPRAk5ACFywqndfpuBMcVz6K/12RtMALmdfGi+GpgO+lXQe3PiGwHtV5wXFRStQwg29XySZZo6tOyvshTSJ1uafhX53S93r5MaqDxJrR0UNGr2VYdKiAm1jYIYQm84v/mEbSAGjjBwEgS1PHlzM72I96JadXzfV9Fmsd5pHlfoLxEqXe6hBJAAQS99CcpwPDnaVA9UZUHA== +X-Message-Delivery: Vj0xLjE7dXM9MDtsPTA7YT0wO0Q9MTtHRD0xO1NDTD0tMQ== +X-Microsoft-Antispam-Message-Info: + =?utf-8?B?VjZIQkpKR05oRUo1Vzc0YTBDUW52S0lsYkJSMGRzY0hJMnRMOWdyRGowcGpk?= + =?utf-8?B?SUJLSDRPaStzakpJUHlaWVFnNWpBSGRsZ1Z4aEFmaXJOR1ZMUWxTTnQ1SXg1?= + =?utf-8?B?anhFNTJ5RGU2YjRiTWhWK3FvWXBJU29YSWdqM3VvUkZpY21aaW5lSkJ5WWph?= + =?utf-8?B?L2pxclptbVBGdm02emlHT3ZBQ1BHZTcrM0c3NmJ5alJLSGlaYVMvK0hwVmJV?= + =?utf-8?B?eHlTU2grSElBTVY5cXF2d250OXBmQ2pzeEVUWTlSZ1hCc1dEdStXMzFGcWlO?= + =?utf-8?B?VytUeEgyRWl5a2U1Y09VKyt3am9ZQVYrRm1LUkhRRGdKbkFTaHc4RTErQ1c0?= + =?utf-8?B?RjBNVllEVW9UakJIQm5FWWVYd2RuaENZTVJIUkI4RmlheWsyajZmanFCUlpt?= + =?utf-8?B?ZTJYZlg1RGxkbEVlRk0zallRWStiU1Z1QmJlTmtKS3J5MmZuOFk2blRHemEw?= + =?utf-8?B?OVhkUUhWWTAzV2dySnMra1pKMGo1Zy8xSFNuemx4Slg1ckhDcitmVGRHSDBW?= + =?utf-8?B?MFlOMDFtNmRPTDVSL3BGU0VNNWRObGVkUUlRcG9MSUJFeVBFcGtlVENSZmIr?= + =?utf-8?B?V3F6by8vOHBROWplTi9JdWtEVDFwUVZsdVk5djBtN0wzbk04RG56RjRsM1ZH?= + =?utf-8?B?cytsajBZNUNwUXk5SVRFZXhMejN3anYweGpCWkltQ2lwQnA3V1B6UUt0VUw1?= + =?utf-8?B?dXpLQ3hxemNQNWRGWmpqZi9BY2EzOTAwQ3h5RlF2RHQyVG1McWp6N1JXUWRY?= + =?utf-8?B?TjlCRWFmNFhQSitwSTk2cEhPK1N3ZVQxbktlMWFwa05hNGllOVpCc2Q3MUEy?= + =?utf-8?B?TlBHVE9YUE8xRUk3dndyNkFQVlhhN3JIMnUxL25pZ3JaM1hFS0VUOXNqT2NF?= + =?utf-8?B?Y3lFcUM0dDVuOGhTdmJ1RjJJK2sxZGViOUU2SE1DTUZ1c0pSSlNsazdPWHJ5?= + =?utf-8?B?TXo0dUUrZEhqaVpGTHNTUnNUTUl2L2hZeFhoNUVtcmJPQ0lXYnV5Yy8rSXBq?= + =?utf-8?B?bjYwVlBET0ErZkQ4KzJsQmM5b0hUTXJSSWlhdXlNeTZ2a0xlaHp5ZTZRQnox?= + =?utf-8?B?T2h2NkZKNmpLcDg4TCs5ckdoU3d5aEc1Q1FYUFdTOXhxcFJsaTdtZkVuNG1W?= + =?utf-8?B?SkVsN2llT3FpTnB6Q3lMbDR4ZzVzblhLVWw3VkpJblRQQVA4cDd1aGdtbll4?= + =?utf-8?B?U2RWQXplZjRreWhJRnQwWGhWT2pnVmxwTW9hdUxwRE9VaTJqd1lqenh3T2pK?= + =?utf-8?B?R2ZMaDJmNm1lS25TNU56ODFBcnc1TUZQbi9pZ0hnampKNUl0MzVQRG5wenZH?= + =?utf-8?B?dTdrcTA4VXUwZmdNaXBKMnVsY1phOEtLUEZWMzNnUlVxYXhrRDFUN3FFN0lZ?= + =?utf-8?B?MnVzbmhVQ2kvQVkzZ3NBQnNGL0NCNlZTbmV5ZW9FVWg5dUJTbmtaQnNZemRT?= + =?utf-8?B?cDFKUnRPU2VpNnNwM3V5eXJxMy9YbFhPYTRFSkEyTUZjSVlNaFV0UE5RbjhK?= + =?utf-8?B?NjJmckpva2xuaGhYT2Jkb2g1U1NEaFJmQWc5bVhheGZYMXY1b2toaVRPOXNT?= + =?utf-8?B?Y2ZhVjYyY0pnbmw4N3VneVR6bXFoRTlndE9lTzlac0JTRWFKc1BMTmNrNFMx?= + =?utf-8?B?M0lwTXI3STZXcFNmbytNcFB2VzJFSFpLSWFpbjlzcVlVRHk3RTFIUUQzOUlB?= + =?utf-8?B?YnR1eC9jUnVNWlhadktVKzM5MmdmR1pBTXVxK2xzUXZ4MzNUWW5rQXZ4SXMv?= + =?utf-8?B?RnBLUmcwT3FUWENucWtuTWhBQnl5VWFpczNGUnBkQ0ltM2ttMDM1RnFScXFa?= + =?utf-8?B?dEtNNnF4Q1FDS2RqRTRuRkNRUC9JVTdZZ216c3hycC9ZalptbDZNZ25ydWFp?= + =?utf-8?B?Z25qMGFLK1FQYm0vUU40OSt1SVJBTmdPTVNRN2JTVmxLTlRJMkZDeldKYWNx?= + =?utf-8?B?VEJEVHE5ZE9QNWsxZkxrb0pFOEU5cUJvT3ArOUFDMXlZM2N4Smk5ay9qQXEv?= + =?utf-8?B?ZXc3ZjVHMjdkcjBkN1Rodmdyd1JldkFBeDlVblRVbkxrY0xhZkIwVzBpTlNM?= + =?utf-8?B?THAvZ01hS3NVK0dHblFFQ0h6VXYydW1QaUwzM29zcjRYRFJRTU9NZWYxQ2Nw?= + =?utf-8?B?N1liQ3g2ZUtveTdTaW1ZSGovLzNWbWh2bDd6ZXRUR3B3eEYwakVCOS95aEs0?= + =?utf-8?B?NkkzL1dQREVlVHFXWmE4RktDUHFENVQwYW9YWE9LS2hrMzAyVWFXTDZFVkx5?= + =?utf-8?B?cU1nZDkzOTR1dk40SHFIcHRDSVRPajMvSVAyd0JQNDJnaVoxNmhNOFEzdzlj?= + =?utf-8?B?ODdUNXRIVkQvTHYzMytWY2o3UHZkdUNTR1pvSVJvclVCN01EZW5pVXdRUDgx?= + =?utf-8?B?Vmg2aUdlOUJzdXlPdXFlL01raHZSbkRONncyRlFLcGpLUFR4bm9BQXVJMHJC?= + =?utf-8?B?cWdJSFJwZEVkZjZkOTJqZG1FNHdZRWpGdUR6R2hjdHRoMTg1Z2lpeGpnZzlH?= + =?utf-8?B?Um5WOEJINFBFM3Evdmt4VVRCQnAwd2xBRGVralpwRnV0eUhJNTluQzFLQXI2?= + =?utf-8?B?NXI4amV3c0ZRZEZLRjE1ZEQ3aW90Y1I0K3NPN3ZoVyt1UVdzWUpQUGh1b25N?= + =?utf-8?Q?amuRKzTLQzIrlx9Vmv+SjIosxogY=3D?= +MIME-Version: 1.0 + +------=_Part_1041327705_575167926.1652070638478 +Content-Type: multipart/related; + boundary="----=_Part_1041327706_445426653.1652070638478" + +------=_Part_1041327706_445426653.1652070638478 +Content-Type: text/html;charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +Here goes the original email content + +------=_Part_1041327706_445426653.1652070638478-- + +------=_Part_1041327705_575167926.1652070638478-- + +--31A9507D-D0B3-4DCD-AFBB-413468892CFE-- diff --git a/tests/example_emls/yahoo_complaint.eml b/tests/example_emls/yahoo_complaint.eml new file mode 100644 index 00000000..a543024a --- /dev/null +++ b/tests/example_emls/yahoo_complaint.eml @@ -0,0 +1,157 @@ +X-SimpleLogin-Client-IP: 66.163.186.21 +Received-SPF: None (mailfrom) identity=mailfrom; client-ip=66.163.186.21; + helo=sonic326-46.consmr.mail.ne1.yahoo.com; + envelope-from=feedback@arf.mail.yahoo.com; receiver= +Received: from sonic326-46.consmr.mail.ne1.yahoo.com + (sonic326-46.consmr.mail.ne1.yahoo.com [66.163.186.21]) + (using TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 (128/128 bits) + key-exchange ECDHE (P-256) server-signature RSA-PSS (2048 bits) + server-digest SHA256) + (No client certificate requested) + by prod4.simplelogin.co (Postfix) with ESMTPS id 160E19C47C + for <{{ postmaster }}>; Sun, 8 May 2022 13:31:32 +0000 (UTC) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arf.mail.yahoo.com; + s=arf; t=1652016690; bh=y3TXlG8d2nUmz+Mm6gBEX1p1y2rwlM+LRC89Bp+HwGo=; + h=Date:From:To:Subject:From:Subject:Reply-To; + b=HyuY58LSzfkdH9FynjNWEl6QJeeImKRbIzrnR64sY/ggFD6fF9w1/fpXDmJ8RHpB/72llGb8nkVJkn/TK+adBCZvw4Y0SC2m8qbn6BdaC5kvAWkN6VUxvQWFMWTptAmeX+UUxY2hjEXLZQwNUd4nvvhZkbdyzw5wFSpYX0hnxAA= +X-SONIC-DKIM-SIGN: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=s2048; + t=1652016690; bh=0SlXAOx+1D8SxkBJpASrTwUGjphtzchFZOSJr0X+U2m=; + h=X-Sonic-MF:Date:From:To:Subject:From:Subject; + b=smqcDrz5jxsmGycWk9tNncLBjcQIqBnZmsQzkJ6g8fyhQw2e30y05iTnsOBTr0S9qTPK3I2JBv0P73TH7vDAnZAnaewzj9Dymw7Z+UxXKdrPBf/tD8RGw9cX6C0eb7GUjHvbvXS03IkSGnvOPPCXLsTDXYOTflcU7A0A2L+cS9ogEBl/4AFwBf/z+lcMH20h2dZ6+wPtqPCgRY1Hf45cv4gfHrFG0a18n3BBq0doCA4cRTXeeuv06fqsUCk2GF6z0mm3YWu+umcUs16QmgjHKhy4SJHvTZfx4zFBxQEOM3hvBzriL5g0D3Rg71CdkI8TVqsyXS1YWVSQFakAw0hM+A== +X-Sonic-MF: feedback@arf.mail.yahoo.com +Received: from sonic.gate.mail.ne1.yahoo.com by + sonic326.consmr.mail.ne1.yahoo.com with HTTP; Sun, 8 May 2022 13:31:30 +0000 +Date: Sun, 8 May 2022 13:31:28 +0000 (UTC) +From: Yahoo! Mail AntiSpam Feedback +To: {{ postmaster }} +Message-ID: + <1486688083.18136997.1652016688605@chakraconsumer2.asd.mail.ne1.yahoo.com> +Subject: Original subject +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=feedback-report; + boundary="----=_Part_18136996_1734597748.1652016688604" +X-Yahoo-Newman-Property: cfl +X-Yahoo-Newman-Id: cfl-test +X-Spamd-Result: default: False [-0.65 / 13.00]; + DMARC_POLICY_ALLOW(-0.50)[yahoo.com,reject]; + R_DKIM_ALLOW(-0.20)[arf.mail.yahoo.com:s=arf]; + SUBJ_ALL_CAPS(0.15)[2]; + MIME_GOOD(-0.10)[text/plain,multipart/alternative]; + R_SPF_NA(0.00)[no SPF record]; + FROM_EQ_ENVFROM(0.00)[]; + MIME_TRACE(0.00)[0:~,1:+,2:~,3:+,4:~,5:+,6:+,7:~]; + RCVD_TLS_LAST(0.00)[]; + RCVD_IN_DNSWL_NONE(0.00)[66.163.186.21:from]; + ASN(0.00)[asn:36646, ipnet:66.163.184.0/21, country:US]; + ARC_NA(0.00)[]; + DKIM_TRACE(0.00)[arf.mail.yahoo.com:+]; + MID_RHS_MATCH_FROMTLD(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + FROM_HAS_DN(0.00)[]; + RCVD_COUNT_TWO(0.00)[2]; + TO_DN_NONE(0.00)[]; + RCPT_COUNT_ONE(0.00)[1]; + NEURAL_SPAM(0.00)[0.429]; + DWL_DNSWL_NONE(0.00)[yahoo.com:dkim] +X-Rspamd-Queue-Id: 160E19C47C +X-Rspamd-Server: prod4 +Content-Transfer-Encoding: 7bit + +------=_Part_18136996_1734597748.1652016688604 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +This is an email abuse report for an email message from simplelogin.co on Sun, 8 May 2022 11:12:35 +0000 + +------=_Part_18136996_1734597748.1652016688604 +Content-Type: message/feedback-report +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Feedback-Type: abuse +User-Agent: Yahoo!-Mail-Feedback/2.0 +Version: 0.1 +Original-Mail-From: + <{{ return_path }}> +Original-Rcpt-To: {{ rcpt }} +Received-Date: Sun, 8 May 2022 11:12:35 +0000 +Reported-Domain: simplelogin.co +Authentication-Results: authentication result string is not available + + +------=_Part_18136996_1734597748.1652016688604 +Content-Type: message/rfc822 +Content-Disposition: inline + +Received: from 10.217.151.74 + by atlas316.free.mail.ne1.yahoo.com with HTTPS; + Sun, 8 May 2022 11:12:34 +0000 +Return-Path: + <{{ return_path }}> +X-Originating-Ip: [176.129.238.160] +Received-SPF: pass (domain of simplelogin.co designates 176.119.200.160 as + permitted sender) +Authentication-Results: atlas316.free.mail.ne1.yahoo.com; + dkim=pass header.i=@simplelogin.co header.s=dkim; + spf=pass smtp.mailfrom=simplelogin.co; + dmarc=pass(p=QUARANTINE) header.from=simplelogin.co; +X-Apparently-To: syn_flood91@yahoo.com; Sun, 8 May 2022 11:12:35 +0000 +X-YMailISG: 5XbMksQWLDvXV9CBjagtqIT6OTC44ku5XiuZJQp_W6hhWfR. + .wUIhFV6vRR_JeMUxC0ZAvugteAP2pe.bqk06ovvYnhJMg_HTvcmfVltbWxQ + tK7xNSs8D2PWQdyDDzB3rdFdIIfSrQnDTGjP2xpTAqLQk3IXSuUBX7s4f8uA + WUELPWj36_Xtqrwyj.ya4Ezw_ePzPhZGmMdCsbz2H5Jh45TLbk5HhL.TDDbH + 9Dz__HKLUC8acH0hu1vrPvo1ljzwbl_0cqlj10qMIChpB51XVDtyNA_WgWvE + QL1hFHS0tScfRT0xATM8w8FJv1eA0ODjakDtTRgmaWBTphzeoR.FyTBj14y5 + burx6lkUqipfP7UZpNmcNDYHQdTEmdGa8JDZMX.lpM5IMOhkByIQuoTN4.Cx + 8qz9kb.o0DqxqNRgn4_fRRAoSn1xejDbzZMu.SWSvJ1KJwAfLtep37ISqNKl + yeBeDJFMnHUjRD8B2wBB46zq4ngHFWjBGkAGQVBssLzj594FXg13aO.TnJU7 + WJ_cUSzoaH9HjgYDTi4.1x68jVxpZIEdhDe7pjLCUL2ugWdar9S7pFlyKWfa + iTH8yQ10NXtLCwGpJ.0kgZH2WXJgyJmrq0a3j63skib7WJYtKOXfsbHV8b9e + WxClOETCe03PtdD6G2sjEJSNFyTH_Qzzq6_21PO6kjmnEnBbibAnkiJbGhIJ + kOSqyp_vFqstpd38vtt7iLI8L3PkyZDQXS0hB1ZCOsZqBDGJXAoWFRBtxMSd + rMVkdvB6r8xJtn.1JrV1hpX4yRbCuEnCCPcwtGamlpyq5LG6YanKUVB868KF + UuZ4AHFwi.m_FYHalwtfCaArtWzYybl2nQQLjPbnXxqNvfwKt3ATKFEO40ZV + w1Ri7y.cO__09.eQHKIUNgMNeWgt.luD3thsEl0yz_ThzrCEkXDB1xAPNnLV + tb03RulEB0xNauYTuWgKR8WJzkO4LuXMlzNAAYBQLQy_t0GoezAs7Z4oq.CH + EfTK88cDJ7j7dXcXBi7q6g1NBZT3tyd9Bfn2DVdFaWAjWV9Lb8tir6J43MDP + byTrZ_zJxTWKgafhOxL0gZbd5xIEZ1eHHeQO5pVZlN6FR1awozFgS4NcZu5u + 5qRtn6zHo3zNe9ORwwxqlHAEJR_5I09WYSdmTxh2QkkDQLjSlwUNV4K8jxdH + L4ePIzNCQCt_bsGoG3uPXl8jtPD4sUWGY1lCeKAm.AHgZ.pSXXypMUpq4y14 + NihY89H61y5ZXo4Zd77shda_ +Received: from 176.119.200.160 (EHLO mail-200160.simplelogin.co) + by 10.217.151.74 with SMTPs + (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256); + Sun, 08 May 2022 11:12:34 +0000 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=simplelogin.co; + s=dkim; t=1652008349; h=From:To:Subject:Message-ID:Date; + bh=9HnrBUpZUe8OSXqTw1qF667IwLtHI8DqiyD0yAovIO4=; + b=PsxiMydvEQveb20xgUvvq3DhxlLyqqoPW7sC8d/pAm8tj7T2O+7z5xxR6vVbgz823Bglzc + djb3pRvNLgHnTozC+FiFOF8nVlWGybosn5oRfmNGkF9bhr0bJmfcDhiuC/tOaZKkod2lbf + jQ8bqMZhCsN/xVpkMqJdNJefdkj3dP4= +MIME-Version: 1.0 +Date: Sun, 8 May 2022 04:11:42 -0700 +Message-ID: + +Subject: MF +Content-Type: multipart/alternative; boundary="0000000000006dd95f05de7e2a70" +Content-Transfer-Encoding: 7bit +X-SimpleLogin-Type: Forward +X-SimpleLogin-EmailLog-ID: 41263490 +X-SimpleLogin-Envelope-From: {{ sender }} +X-SimpleLogin-Envelope-To: {{ rcpt }} +From: {{ sender }} +To: {{ rcpt_comma_list }} +List-Unsubscribe: +X-SimpleLogin-Want-Signing: yes +Content-Length: 473 + +--0000000000006dd95f05de7e2a70 +Content-Type: text/plain; charset="UTF-8" + +Here goes the original email content + +--0000000000006dd95f05de7e2a70-- + + +------=_Part_18136996_1734597748.1652016688604-- diff --git a/tests/handler/test_provider_complaints.py b/tests/handler/test_provider_complaints.py index 00b9a6a5..b222a752 100644 --- a/tests/handler/test_provider_complaints.py +++ b/tests/handler/test_provider_complaints.py @@ -1,51 +1,46 @@ -import email from email.message import Message -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText import pytest - from app.config import ( ALERT_COMPLAINT_FORWARD_PHASE, ALERT_COMPLAINT_REPLY_PHASE, ALERT_COMPLAINT_TRANSACTIONAL_PHASE, + POSTMASTER, ) from app.db import Session -from app.email import headers +from app.email import headers, status from app.handler.provider_complaint import ( handle_hotmail_complaint, handle_yahoo_complaint, ) from app.models import Alias, ProviderComplaint, SentAlert -from tests.utils import create_new_user +from tests.utils import create_new_user, load_eml_file origins = [ - [handle_yahoo_complaint, "yahoo", 6], - [handle_hotmail_complaint, "hotmail", 3], + [handle_yahoo_complaint, "yahoo"], + [handle_hotmail_complaint, "hotmail"], ] -def prepare_complaint(message: Message, part_num: int) -> Message: - complaint = MIMEMultipart("related") - # When walking, part 0 is the full message so we -1, and we want to be part N so -1 again - for i in range(part_num - 2): - document = MIMEText("text", "plain") - document.set_payload(f"Part {i}") - complaint.attach(document) - complaint.attach(message) - - return email.message_from_bytes(complaint.as_bytes()) +def prepare_complaint( + provider_name: str, rcpt_address: str, sender_address: str +) -> Message: + return load_eml_file( + f"{provider_name}_complaint.eml", + { + "postmaster": POSTMASTER, + "return_path": "sl.something.other@simplelogin.co", + "rcpt": rcpt_address, + "sender": sender_address, + "rcpt_comma_list": f"{rcpt_address},other_rcpt@somwhere.net", + }, + ) -@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_provider_to_user(flask_client, handle_ftor, provider, part_num): +@pytest.mark.parametrize("handle_ftor,provider", origins) +def test_provider_to_user(flask_client, handle_ftor, provider): user = create_new_user() - original_message = Message() - original_message[headers.TO] = user.email - original_message[headers.FROM] = "nobody@nowhere.net" - original_message.set_payload("Contents") - - complaint = prepare_complaint(original_message, part_num) + complaint = prepare_complaint(provider, user.email, "nobody@nowhere.net") assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 @@ -54,17 +49,12 @@ def test_provider_to_user(flask_client, handle_ftor, provider, part_num): assert alerts[0].alert_type == f"{ALERT_COMPLAINT_TRANSACTIONAL_PHASE}_{provider}" -@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_provider_forward_phase(flask_client, handle_ftor, provider, part_num): +@pytest.mark.parametrize("handle_ftor,provider", origins) +def test_provider_forward_phase(flask_client, handle_ftor, provider): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() - original_message = Message() - original_message[headers.TO] = "nobody@nowhere.net" - original_message[headers.FROM] = alias.email - original_message.set_payload("Contents") - - complaint = prepare_complaint(original_message, part_num) + complaint = prepare_complaint(provider, "nobody@nowhere.net", alias.email) assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 1 @@ -73,8 +63,8 @@ def test_provider_forward_phase(flask_client, handle_ftor, provider, part_num): assert alerts[0].alert_type == f"{ALERT_COMPLAINT_REPLY_PHASE}_{provider}" -@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_provider_reply_phase(flask_client, handle_ftor, provider, part_num): +@pytest.mark.parametrize("handle_ftor,provider", origins) +def test_provider_reply_phase(flask_client, handle_ftor, provider): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() @@ -83,7 +73,7 @@ def test_provider_reply_phase(flask_client, handle_ftor, provider, part_num): original_message[headers.FROM] = "no@no.no" original_message.set_payload("Contents") - complaint = prepare_complaint(original_message, part_num) + complaint = prepare_complaint(provider, alias.email, "no@no.no") assert handle_ftor(complaint) found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 diff --git a/tests/test.env b/tests/test.env index 8189945f..a4ffe350 100644 --- a/tests/test.env +++ b/tests/test.env @@ -60,3 +60,5 @@ DMARC_CHECK_ENABLED=true PROTON_CLIENT_ID=to_fill PROTON_CLIENT_SECRET=to_fill PROTON_BASE_URL=https://localhost/api + +POSTMASTER=postmaster@simplelogin.co \ No newline at end of file