Create signed email addresses for VERP emails
This commit is contained in:
parent
110f2f2f2c
commit
db06ce0ae6
|
@ -74,7 +74,6 @@ MONITORING_EMAIL = os.environ.get("MONITORING_EMAIL")
|
||||||
# VERP: mail_from set to BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX
|
# VERP: mail_from set to BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX
|
||||||
BOUNCE_PREFIX = os.environ.get("BOUNCE_PREFIX") or "bounce+"
|
BOUNCE_PREFIX = os.environ.get("BOUNCE_PREFIX") or "bounce+"
|
||||||
BOUNCE_SUFFIX = os.environ.get("BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
|
BOUNCE_SUFFIX = os.environ.get("BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
|
||||||
BOUNCE_EMAIL = BOUNCE_PREFIX + "{}" + BOUNCE_SUFFIX
|
|
||||||
|
|
||||||
# Used for VERP during reply phase. It's similar to BOUNCE_PREFIX.
|
# Used for VERP during reply phase. It's similar to BOUNCE_PREFIX.
|
||||||
# It's needed when sending emails from custom domain to respect DMARC.
|
# It's needed when sending emails from custom domain to respect DMARC.
|
||||||
|
@ -93,9 +92,6 @@ TRANSACTIONAL_BOUNCE_PREFIX = (
|
||||||
TRANSACTIONAL_BOUNCE_SUFFIX = (
|
TRANSACTIONAL_BOUNCE_SUFFIX = (
|
||||||
os.environ.get("TRANSACTIONAL_BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
|
os.environ.get("TRANSACTIONAL_BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
|
||||||
)
|
)
|
||||||
TRANSACTIONAL_BOUNCE_EMAIL = (
|
|
||||||
TRANSACTIONAL_BOUNCE_PREFIX + "{}" + TRANSACTIONAL_BOUNCE_SUFFIX
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"])
|
MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"])
|
||||||
|
@ -169,6 +165,8 @@ DB_URI = os.environ["DB_URI"]
|
||||||
|
|
||||||
# Flask secret
|
# Flask secret
|
||||||
FLASK_SECRET = os.environ["FLASK_SECRET"]
|
FLASK_SECRET = os.environ["FLASK_SECRET"]
|
||||||
|
if not FLASK_SECRET:
|
||||||
|
raise RuntimeError("FLASK_SECRET is empty. Please define it.")
|
||||||
SESSION_COOKIE_NAME = "slapp"
|
SESSION_COOKIE_NAME = "slapp"
|
||||||
MAILBOX_SECRET = FLASK_SECRET + "mailbox"
|
MAILBOX_SECRET = FLASK_SECRET + "mailbox"
|
||||||
CUSTOM_ALIAS_SECRET = FLASK_SECRET + "custom_alias"
|
CUSTOM_ALIAS_SECRET = FLASK_SECRET + "custom_alias"
|
||||||
|
@ -426,6 +424,18 @@ ZENDESK_ENABLED = "ZENDESK_ENABLED" in os.environ
|
||||||
|
|
||||||
DMARC_CHECK_ENABLED = "DMARC_CHECK_ENABLED" in os.environ
|
DMARC_CHECK_ENABLED = "DMARC_CHECK_ENABLED" in os.environ
|
||||||
|
|
||||||
|
# Bounces can happen after 5 days
|
||||||
|
VERP_MESSAGE_LIFETIME = 5 * 865400
|
||||||
|
VERP_PREFIX = os.environ.get("VERP_PREFIX") or "sl"
|
||||||
|
# Generate with python3 -c 'import secrets; print(secrets.token_hex(28))'
|
||||||
|
VERP_EMAIL_SECRET = os.environ.get("VERP_EMAIL_SECRET") or (
|
||||||
|
FLASK_SECRET + "pleasegenerateagoodrandomtoken"
|
||||||
|
)
|
||||||
|
if len(VERP_EMAIL_SECRET) < 32:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Please, set VERP_EMAIL_SECRET to a random string at least 32 chars long"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_allowed_redirect_domains() -> List[str]:
|
def get_allowed_redirect_domains() -> List[str]:
|
||||||
allowed_domains = sl_getenv("ALLOWED_REDIRECT_DOMAINS", list)
|
allowed_domains = sl_getenv("ALLOWED_REDIRECT_DOMAINS", list)
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
import base64
|
import base64
|
||||||
import enum
|
import enum
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import quopri
|
import quopri
|
||||||
import random
|
import random
|
||||||
|
@ -46,13 +48,15 @@ from app.config import (
|
||||||
LANDING_PAGE_URL,
|
LANDING_PAGE_URL,
|
||||||
EMAIL_DOMAIN,
|
EMAIL_DOMAIN,
|
||||||
ALERT_DIRECTORY_DISABLED_ALIAS_CREATION,
|
ALERT_DIRECTORY_DISABLED_ALIAS_CREATION,
|
||||||
TRANSACTIONAL_BOUNCE_EMAIL,
|
|
||||||
ALERT_SPF,
|
ALERT_SPF,
|
||||||
ALERT_INVALID_TOTP_LOGIN,
|
ALERT_INVALID_TOTP_LOGIN,
|
||||||
TEMP_DIR,
|
TEMP_DIR,
|
||||||
ALIAS_AUTOMATIC_DISABLE,
|
ALIAS_AUTOMATIC_DISABLE,
|
||||||
RSPAMD_SIGN_DKIM,
|
RSPAMD_SIGN_DKIM,
|
||||||
NOREPLY,
|
NOREPLY,
|
||||||
|
VERP_PREFIX,
|
||||||
|
VERP_MESSAGE_LIFETIME,
|
||||||
|
VERP_EMAIL_SECRET,
|
||||||
)
|
)
|
||||||
from app.db import Session
|
from app.db import Session
|
||||||
from app.dns_utils import get_mx_domains
|
from app.dns_utils import get_mx_domains
|
||||||
|
@ -71,6 +75,7 @@ from app.models import (
|
||||||
IgnoreBounceSender,
|
IgnoreBounceSender,
|
||||||
InvalidMailboxDomain,
|
InvalidMailboxDomain,
|
||||||
DmarcCheckResult,
|
DmarcCheckResult,
|
||||||
|
VerpType,
|
||||||
SpamdResult,
|
SpamdResult,
|
||||||
SPFCheckResult,
|
SPFCheckResult,
|
||||||
)
|
)
|
||||||
|
@ -324,7 +329,7 @@ def send_email(
|
||||||
|
|
||||||
# use a different envelope sender for each transactional email (aka VERP)
|
# use a different envelope sender for each transactional email (aka VERP)
|
||||||
sl_sendmail(
|
sl_sendmail(
|
||||||
TRANSACTIONAL_BOUNCE_EMAIL.format(transaction.id),
|
generate_verp_email(VerpType.transactional, transaction.id),
|
||||||
to_email,
|
to_email,
|
||||||
msg,
|
msg,
|
||||||
retries=retries,
|
retries=retries,
|
||||||
|
@ -1466,3 +1471,49 @@ def get_spamd_result(msg: Message) -> Optional[SpamdResult]:
|
||||||
|
|
||||||
newrelic.agent.record_custom_event("SpamdCheck", spamd_result.event_data())
|
newrelic.agent.record_custom_event("SpamdCheck", spamd_result.event_data())
|
||||||
return spamd_result
|
return spamd_result
|
||||||
|
|
||||||
|
|
||||||
|
def generate_verp_email(
|
||||||
|
verp_type: VerpType, object_id: int, sender_domain: Optional[str]
|
||||||
|
) -> str:
|
||||||
|
# Encoded as a list to minimize size of email address
|
||||||
|
data = [verp_type.bounce_forward.value, object_id, int(time.time())]
|
||||||
|
json_payload = json.dumps(data).encode("utf-8")
|
||||||
|
# Signing without itsdangereous because it uses base64 that includes +/= symbols and lower and upper case letters.
|
||||||
|
# We need to encode in base32
|
||||||
|
payload_hmac = hmac.new(
|
||||||
|
VERP_EMAIL_SECRET.encode("utf-8"), json_payload, "shake128"
|
||||||
|
).digest()
|
||||||
|
encoded_payload = base64.b32encode(json_payload).rstrip(b"=").decode("utf-8")
|
||||||
|
encoded_signature = base64.b32encode(payload_hmac).rstrip(b"=").decode("utf-8")
|
||||||
|
return "{}.{}.{}@{}".format(
|
||||||
|
VERP_PREFIX, encoded_payload, encoded_signature, sender_domain or EMAIL_DOMAIN
|
||||||
|
).lower()
|
||||||
|
|
||||||
|
|
||||||
|
# This method processes the email address, checks if it's a signed verp email generated by us to receive bounces
|
||||||
|
# and extracts the type of verp email and associated email log id/transactional email id stored as object_id
|
||||||
|
def get_verp_info_from_email(email: str) -> Optional[Tuple[VerpType, int]]:
|
||||||
|
idx = email.find("@")
|
||||||
|
if idx == -1:
|
||||||
|
return None
|
||||||
|
username = email[:idx]
|
||||||
|
fields = username.split(".")
|
||||||
|
if len(fields) != 3 or fields[0] != VERP_PREFIX:
|
||||||
|
return None
|
||||||
|
padding = 8 - (len(fields[1]) % 8)
|
||||||
|
payload = base64.b32decode(fields[1].encode("utf-8").upper() + (b"=" * padding))
|
||||||
|
padding = 8 - (len(fields[2]) % 8)
|
||||||
|
signature = base64.b32decode(fields[2].encode("utf-8").upper() + (b"=" * padding))
|
||||||
|
expected_signature = hmac.new(
|
||||||
|
VERP_EMAIL_SECRET.encode("utf-8"), payload, "shake128"
|
||||||
|
).digest()
|
||||||
|
if expected_signature != signature:
|
||||||
|
return None
|
||||||
|
data = json.loads(payload)
|
||||||
|
# verp type, object_id, time
|
||||||
|
if len(data) != 3:
|
||||||
|
return None
|
||||||
|
if data[2] > time.time() + VERP_MESSAGE_LIFETIME:
|
||||||
|
return None
|
||||||
|
return VerpType(data[0]), data[1]
|
||||||
|
|
|
@ -294,6 +294,12 @@ class SpamdResult:
|
||||||
return {"header": "present", "dmarc": self.dmarc, "spf": self.spf}
|
return {"header": "present", "dmarc": self.dmarc, "spf": self.spf}
|
||||||
|
|
||||||
|
|
||||||
|
class VerpType(EnumE):
|
||||||
|
bounce_forward = 0
|
||||||
|
bounce_reply = 1
|
||||||
|
transactional = 2
|
||||||
|
|
||||||
|
|
||||||
class Hibp(Base, ModelMixin):
|
class Hibp(Base, ModelMixin):
|
||||||
__tablename__ = "hibp"
|
__tablename__ = "hibp"
|
||||||
name = sa.Column(sa.String(), nullable=False, unique=True, index=True)
|
name = sa.Column(sa.String(), nullable=False, unique=True, index=True)
|
||||||
|
|
|
@ -72,7 +72,6 @@ from app.config import (
|
||||||
PGP_SENDER_PRIVATE_KEY,
|
PGP_SENDER_PRIVATE_KEY,
|
||||||
ALERT_BOUNCE_EMAIL_REPLY_PHASE,
|
ALERT_BOUNCE_EMAIL_REPLY_PHASE,
|
||||||
NOREPLY,
|
NOREPLY,
|
||||||
BOUNCE_EMAIL,
|
|
||||||
BOUNCE_PREFIX,
|
BOUNCE_PREFIX,
|
||||||
BOUNCE_SUFFIX,
|
BOUNCE_SUFFIX,
|
||||||
TRANSACTIONAL_BOUNCE_PREFIX,
|
TRANSACTIONAL_BOUNCE_PREFIX,
|
||||||
|
@ -131,6 +130,8 @@ from app.email_utils import (
|
||||||
get_mailbox_bounce_info,
|
get_mailbox_bounce_info,
|
||||||
save_email_for_debugging,
|
save_email_for_debugging,
|
||||||
get_spamd_result,
|
get_spamd_result,
|
||||||
|
generate_verp_email,
|
||||||
|
get_verp_info_from_email,
|
||||||
)
|
)
|
||||||
from app.errors import (
|
from app.errors import (
|
||||||
NonReverseAliasInReplyPhase,
|
NonReverseAliasInReplyPhase,
|
||||||
|
@ -963,7 +964,7 @@ def forward_email_to_mailbox(
|
||||||
try:
|
try:
|
||||||
sl_sendmail(
|
sl_sendmail(
|
||||||
# use a different envelope sender for each forward (aka VERP)
|
# use a different envelope sender for each forward (aka VERP)
|
||||||
BOUNCE_EMAIL.format(email_log.id),
|
generate_verp_email(VerpType.bounce_forward, email_log.id),
|
||||||
mailbox.email,
|
mailbox.email,
|
||||||
msg,
|
msg,
|
||||||
envelope.mail_options,
|
envelope.mail_options,
|
||||||
|
@ -1246,12 +1247,9 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
|
||||||
if should_add_dkim_signature(alias_domain):
|
if should_add_dkim_signature(alias_domain):
|
||||||
add_dkim_signature(msg, alias_domain)
|
add_dkim_signature(msg, alias_domain)
|
||||||
|
|
||||||
# generate a mail_from for VERP
|
|
||||||
verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}"
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
sl_sendmail(
|
sl_sendmail(
|
||||||
verp_mail_from,
|
generate_verp_email(VerpType.bounce_reply, email_log.id, alias_domain),
|
||||||
contact.website_email,
|
contact.website_email,
|
||||||
msg,
|
msg,
|
||||||
envelope.mail_options,
|
envelope.mail_options,
|
||||||
|
@ -2092,11 +2090,13 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
|
||||||
return status.E202
|
return status.E202
|
||||||
|
|
||||||
|
|
||||||
def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to):
|
def handle_transactional_bounce(
|
||||||
|
envelope: Envelope, msg, rcpt_to, transactional_id=None
|
||||||
|
):
|
||||||
LOG.d("handle transactional bounce sent to %s", rcpt_to)
|
LOG.d("handle transactional bounce sent to %s", rcpt_to)
|
||||||
|
|
||||||
# parse the TransactionalEmail
|
# parse the TransactionalEmail
|
||||||
transactional_id = parse_id_from_bounce(rcpt_to)
|
transactional_id = transactional_id or parse_id_from_bounce(rcpt_to)
|
||||||
transactional = TransactionalEmail.get(transactional_id)
|
transactional = TransactionalEmail.get(transactional_id)
|
||||||
|
|
||||||
# a transaction might have been deleted in delete_logs()
|
# a transaction might have been deleted in delete_logs()
|
||||||
|
@ -2285,15 +2285,18 @@ def handle(envelope: Envelope, msg: Message) -> str:
|
||||||
return handle_unsubscribe(envelope, msg)
|
return handle_unsubscribe(envelope, msg)
|
||||||
|
|
||||||
# region mail sent to VERP
|
# region mail sent to VERP
|
||||||
|
verp_info = get_verp_info_from_email(rcpt_tos[0])
|
||||||
|
|
||||||
# sent to transactional VERP. Either bounce emails or out-of-office
|
# sent to transactional VERP. Either bounce emails or out-of-office
|
||||||
if (
|
if (
|
||||||
len(rcpt_tos) == 1
|
len(rcpt_tos) == 1
|
||||||
and rcpt_tos[0].startswith(TRANSACTIONAL_BOUNCE_PREFIX)
|
and rcpt_tos[0].startswith(TRANSACTIONAL_BOUNCE_PREFIX)
|
||||||
and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
|
and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
|
||||||
):
|
) or (verp_info and verp_info[0] == VerpType.transactional):
|
||||||
if is_bounce(envelope, msg):
|
if is_bounce(envelope, msg):
|
||||||
handle_transactional_bounce(envelope, msg, rcpt_tos[0])
|
handle_transactional_bounce(
|
||||||
|
envelope, msg, rcpt_tos[0], verp_info and verp_info[1]
|
||||||
|
)
|
||||||
return status.E205
|
return status.E205
|
||||||
elif is_automatic_out_of_office(msg):
|
elif is_automatic_out_of_office(msg):
|
||||||
LOG.d(
|
LOG.d(
|
||||||
|
@ -2308,8 +2311,8 @@ def handle(envelope: Envelope, msg: Message) -> str:
|
||||||
len(rcpt_tos) == 1
|
len(rcpt_tos) == 1
|
||||||
and rcpt_tos[0].startswith(BOUNCE_PREFIX)
|
and rcpt_tos[0].startswith(BOUNCE_PREFIX)
|
||||||
and rcpt_tos[0].endswith(BOUNCE_SUFFIX)
|
and rcpt_tos[0].endswith(BOUNCE_SUFFIX)
|
||||||
):
|
) or (verp_info and verp_info[0] == VerpType.bounce_forward):
|
||||||
email_log_id = parse_id_from_bounce(rcpt_tos[0])
|
email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0])
|
||||||
email_log = EmailLog.get(email_log_id)
|
email_log = EmailLog.get(email_log_id)
|
||||||
|
|
||||||
if not email_log:
|
if not email_log:
|
||||||
|
@ -2324,10 +2327,12 @@ def handle(envelope: Envelope, msg: Message) -> str:
|
||||||
raise VERPForward
|
raise VERPForward
|
||||||
|
|
||||||
# sent to reply VERP, can be either bounce or out-of-office
|
# sent to reply VERP, can be either bounce or out-of-office
|
||||||
if len(rcpt_tos) == 1 and rcpt_tos[0].startswith(
|
if (
|
||||||
f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+"
|
len(rcpt_tos) == 1
|
||||||
|
and rcpt_tos[0].startswith(f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+")
|
||||||
|
or (verp_info and verp_info[0] == VerpType.bounce_reply)
|
||||||
):
|
):
|
||||||
email_log_id = parse_id_from_bounce(rcpt_tos[0])
|
email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0])
|
||||||
email_log = EmailLog.get(email_log_id)
|
email_log = EmailLog.get(email_log_id)
|
||||||
|
|
||||||
if not email_log:
|
if not email_log:
|
||||||
|
@ -2346,12 +2351,13 @@ def handle(envelope: Envelope, msg: Message) -> str:
|
||||||
)
|
)
|
||||||
|
|
||||||
# iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias
|
# iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias
|
||||||
|
verp_info = get_verp_info_from_email(mail_from[0])
|
||||||
if (
|
if (
|
||||||
len(rcpt_tos) == 1
|
len(rcpt_tos) == 1
|
||||||
and mail_from.startswith(BOUNCE_PREFIX)
|
and mail_from.startswith(BOUNCE_PREFIX)
|
||||||
and mail_from.endswith(BOUNCE_SUFFIX)
|
and mail_from.endswith(BOUNCE_SUFFIX)
|
||||||
):
|
) or (verp_info and verp_info[0] == VerpType.bounce_forward):
|
||||||
email_log_id = parse_id_from_bounce(mail_from)
|
email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(mail_from)
|
||||||
email_log = EmailLog.get(email_log_id)
|
email_log = EmailLog.get(email_log_id)
|
||||||
alias = Alias.get_by(email=rcpt_tos[0])
|
alias = Alias.get_by(email=rcpt_tos[0])
|
||||||
LOG.w(
|
LOG.w(
|
||||||
|
|
|
@ -5,7 +5,7 @@ from email.message import EmailMessage
|
||||||
import arrow
|
import arrow
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR
|
from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, ROOT_DIR
|
||||||
from app.db import Session
|
from app.db import Session
|
||||||
from app.email_utils import (
|
from app.email_utils import (
|
||||||
get_email_domain_part,
|
get_email_domain_part,
|
||||||
|
@ -37,6 +37,8 @@ from app.email_utils import (
|
||||||
get_mailbox_bounce_info,
|
get_mailbox_bounce_info,
|
||||||
is_invalid_mailbox_domain,
|
is_invalid_mailbox_domain,
|
||||||
get_spamd_result,
|
get_spamd_result,
|
||||||
|
generate_verp_email,
|
||||||
|
get_verp_info_from_email,
|
||||||
)
|
)
|
||||||
from app.models import (
|
from app.models import (
|
||||||
User,
|
User,
|
||||||
|
@ -47,6 +49,7 @@ from app.models import (
|
||||||
IgnoreBounceSender,
|
IgnoreBounceSender,
|
||||||
InvalidMailboxDomain,
|
InvalidMailboxDomain,
|
||||||
DmarcCheckResult,
|
DmarcCheckResult,
|
||||||
|
VerpType,
|
||||||
)
|
)
|
||||||
|
|
||||||
# flake8: noqa: E101, W191
|
# flake8: noqa: E101, W191
|
||||||
|
@ -739,7 +742,6 @@ def test_should_disable_bounce_consecutive_days(flask_client):
|
||||||
def test_parse_id_from_bounce():
|
def test_parse_id_from_bounce():
|
||||||
assert parse_id_from_bounce("bounces+1234+@local") == 1234
|
assert parse_id_from_bounce("bounces+1234+@local") == 1234
|
||||||
assert parse_id_from_bounce("anything+1234+@local") == 1234
|
assert parse_id_from_bounce("anything+1234+@local") == 1234
|
||||||
assert parse_id_from_bounce(BOUNCE_EMAIL.format(1234)) == 1234
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_queue_id():
|
def test_get_queue_id():
|
||||||
|
@ -823,3 +825,11 @@ def test_dmarc_result_na():
|
||||||
def test_dmarc_result_bad_policy():
|
def test_dmarc_result_bad_policy():
|
||||||
msg = load_eml_file("dmarc_bad_policy.eml")
|
msg = load_eml_file("dmarc_bad_policy.eml")
|
||||||
assert DmarcCheckResult.bad_policy == get_spamd_result(msg).dmarc
|
assert DmarcCheckResult.bad_policy == get_spamd_result(msg).dmarc
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_verp_email():
|
||||||
|
generated_email = generate_verp_email(VerpType.bounce_forward, 1, "somewhere.net")
|
||||||
|
print(generated_email)
|
||||||
|
info = get_verp_info_from_email(generated_email.lower())
|
||||||
|
assert info[0] == VerpType.bounce_forward
|
||||||
|
assert info[1] == 1
|
||||||
|
|
Loading…
Reference in New Issue