Mirror
/
app-MAIL-temp
mirror of https://github.com/simple-login/app.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

disable should_disable() for now

This commit is contained in:
Son Nguyen Kim 2021-07-13 17:06:58 +02:00
parent 7ce83c36b9
commit fb88654d84
2 changed files with 223 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
app/email_utils.py
View File

@ -13,13 +13,12 @@ from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
from email.utils import make_msgid, formatdate, parseaddr from email.utils import make_msgid, formatdate, parseaddr
from smtplib import SMTP, SMTPServerDisconnected from smtplib import SMTP, SMTPServerDisconnected
from typing import Tuple, List, Optional from typing import Optional
import arrow import arrow
import dkim import dkim
import spf import spf
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
from sqlalchemy import func
from validate_email import validate_email from validate_email import validate_email
from app.config import ( from app.config import (
@ -56,7 +55,6 @@ from app.models import (
SLDomain, SLDomain,
Contact, Contact,
Alias, Alias,
EmailLog,
TransactionalEmail, TransactionalEmail,
) )
from app.utils import ( from app.utils import (
@ -975,98 +973,101 @@ def should_disable(alias: Alias) -> bool:
LOG.warning("%s cannot be disabled", alias) LOG.warning("%s cannot be disabled", alias)
return False return False
yesterday = arrow.now().shift(days=-1) # todo: remove
nb_bounced_last_24h = (
db.session.query(EmailLog)
.join(Contact, EmailLog.contact_id == Contact.id)
.filter(
EmailLog.bounced.is_(True),
EmailLog.is_reply.is_(False),
EmailLog.created_at > yesterday,
)
.filter(Contact.alias_id == alias.id)
.count()
)
# if more than 12 bounces in 24h -> disable alias
if nb_bounced_last_24h > 12:
LOG.d("more than 12 bounces in the last 24h, disable alias %s", alias)
return True
# if more than 5 bounces but has bounces last week -> disable alias
elif nb_bounced_last_24h > 5:
one_week_ago = arrow.now().shift(days=-8)
nb_bounced_7d_1d = (
db.session.query(EmailLog)
.join(Contact, EmailLog.contact_id == Contact.id)
.filter(
EmailLog.bounced.is_(True),
EmailLog.is_reply.is_(False),
EmailLog.created_at > one_week_ago,
EmailLog.created_at < yesterday,
)
.filter(Contact.alias_id == alias.id)
.count()
)
if nb_bounced_7d_1d > 1:
LOG.debug(
"more than 5 bounces in the last 24h and more than 1 bounces in the last 7 days, "
"disable alias %s",
alias,
)
return True
else:
# alias level
# if bounces at least 9 days in the last 10 days -> disable alias
query = (
db.session.query(
func.date(EmailLog.created_at).label("date"),
func.count(EmailLog.id).label("count"),
)
.join(Contact, EmailLog.contact_id == Contact.id)
.filter(Contact.alias_id == alias.id)
.filter(
EmailLog.created_at > arrow.now().shift(days=-10),
EmailLog.bounced.is_(True),
EmailLog.is_reply.is_(False),
)
.group_by("date")
)
if query.count() >= 9:
LOG.d(
"Bounces every day for at least 9 days in the last 10 days, disable alias %s",
alias,
)
return True
# account level
query = (
db.session.query(
func.date(EmailLog.created_at).label("date"),
func.count(EmailLog.id).label("count"),
)
.filter(EmailLog.user_id == alias.user_id)
.filter(
EmailLog.created_at > arrow.now().shift(days=-10),
EmailLog.bounced.is_(True),
EmailLog.is_reply.is_(False),
)
.group_by("date")
)
# if an account has more than 10 bounces every day for at least 4 days in the last 10 days, disable alias
date_bounces: List[Tuple[arrow.Arrow, int]] = list(query)
if len(date_bounces) > 4:
if all([v > 10 for _, v in date_bounces]):
LOG.d(
"+10 bounces for +4 days in the last 10 days on %s, disable alias %s",
alias.user,
alias,
)
return True
return False return False
# yesterday = arrow.now().shift(days=-1)
# nb_bounced_last_24h = (
# db.session.query(EmailLog)
# .join(Contact, EmailLog.contact_id == Contact.id)
# .filter(
# EmailLog.bounced.is_(True),
# EmailLog.is_reply.is_(False),
# EmailLog.created_at > yesterday,
# )
# .filter(Contact.alias_id == alias.id)
# .count()
# )
# # if more than 12 bounces in 24h -> disable alias
# if nb_bounced_last_24h > 12:
# LOG.d("more than 12 bounces in the last 24h, disable alias %s", alias)
# return True
#
# # if more than 5 bounces but has bounces last week -> disable alias
# elif nb_bounced_last_24h > 5:
# one_week_ago = arrow.now().shift(days=-8)
# nb_bounced_7d_1d = (
# db.session.query(EmailLog)
# .join(Contact, EmailLog.contact_id == Contact.id)
# .filter(
# EmailLog.bounced.is_(True),
# EmailLog.is_reply.is_(False),
# EmailLog.created_at > one_week_ago,
# EmailLog.created_at < yesterday,
# )
# .filter(Contact.alias_id == alias.id)
# .count()
# )
# if nb_bounced_7d_1d > 1:
# LOG.debug(
# "more than 5 bounces in the last 24h and more than 1 bounces in the last 7 days, "
# "disable alias %s",
# alias,
# )
# return True
# else:
# # alias level
# # if bounces at least 9 days in the last 10 days -> disable alias
# query = (
# db.session.query(
# func.date(EmailLog.created_at).label("date"),
# func.count(EmailLog.id).label("count"),
# )
# .join(Contact, EmailLog.contact_id == Contact.id)
# .filter(Contact.alias_id == alias.id)
# .filter(
# EmailLog.created_at > arrow.now().shift(days=-10),
# EmailLog.bounced.is_(True),
# EmailLog.is_reply.is_(False),
# )
# .group_by("date")
# )
#
# if query.count() >= 9:
# LOG.d(
# "Bounces every day for at least 9 days in the last 10 days, disable alias %s",
# alias,
# )
# return True
#
# # account level
# query = (
# db.session.query(
# func.date(EmailLog.created_at).label("date"),
# func.count(EmailLog.id).label("count"),
# )
# .filter(EmailLog.user_id == alias.user_id)
# .filter(
# EmailLog.created_at > arrow.now().shift(days=-10),
# EmailLog.bounced.is_(True),
# EmailLog.is_reply.is_(False),
# )
# .group_by("date")
# )
#
# # if an account has more than 10 bounces every day for at least 4 days in the last 10 days, disable alias
# date_bounces: List[Tuple[arrow.Arrow, int]] = list(query)
# if len(date_bounces) > 4:
# if all([v > 10 for _, v in date_bounces]):
# LOG.d(
# "+10 bounces for +4 days in the last 10 days on %s, disable alias %s",
# alias.user,
# alias,
# )
# return True
#
# return False
def parse_id_from_bounce(email_address: str) -> int: def parse_id_from_bounce(email_address: str) -> int:
return int(email_address[email_address.find("+") : email_address.rfind("+")]) return int(email_address[email_address.find("+") : email_address.rfind("+")])

261
tests/test_email_utils.py
View File

@ -1,8 +1,6 @@
import email import email
from email.message import EmailMessage from email.message import EmailMessage
import arrow
from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL
from app.email_utils import ( from app.email_utils import (
get_email_domain_part, get_email_domain_part,
@ -24,16 +22,15 @@ from app.email_utils import (
encode_text, encode_text,
EmailEncoding, EmailEncoding,
replace, replace,
should_disable,
decode_text, decode_text,
parse_id_from_bounce, parse_id_from_bounce,
get_queue_id, get_queue_id,
) )
from app.extensions import db from app.extensions import db
from app.models import User, CustomDomain, Alias, Contact, EmailLog from app.models import User, CustomDomain
# flake8: noqa: E101, W191 # flake8: noqa: E101, W191
from tests.utils import login
def test_get_email_domain_part(): def test_get_email_domain_part():
@ -593,139 +590,139 @@ def test_decode_text():
) )
def test_should_disable(flask_client): # def test_should_disable(flask_client):
user = User.create( # user = User.create(
email="a@b.c", # email="a@b.c",
password="password", # password="password",
name="Test User", # name="Test User",
activated=True, # activated=True,
include_sender_in_reverse_alias=True, # include_sender_in_reverse_alias=True,
) # )
alias = Alias.create_new_random(user) # alias = Alias.create_new_random(user)
db.session.commit() # db.session.commit()
#
assert not should_disable(alias) # assert not should_disable(alias)
#
# create a lot of bounce on this alias # # create a lot of bounce on this alias
contact = Contact.create( # contact = Contact.create(
user_id=user.id, # user_id=user.id,
alias_id=alias.id, # alias_id=alias.id,
website_email="contact@example.com", # website_email="contact@example.com",
reply_email="rep@sl.local", # reply_email="rep@sl.local",
commit=True, # commit=True,
) # )
for _ in range(20): # for _ in range(20):
EmailLog.create( # EmailLog.create(
user_id=user.id, # user_id=user.id,
contact_id=contact.id, # contact_id=contact.id,
alias_id=contact.alias_id, # alias_id=contact.alias_id,
commit=True, # commit=True,
bounced=True, # bounced=True,
) # )
#
assert should_disable(alias) # assert should_disable(alias)
#
# should not affect another alias # # should not affect another alias
alias2 = Alias.create_new_random(user) # alias2 = Alias.create_new_random(user)
db.session.commit() # db.session.commit()
assert not should_disable(alias2) # assert not should_disable(alias2)
def test_should_disable_bounces_every_day(flask_client): # def test_should_disable_bounces_every_day(flask_client):
"""if an alias has bounces every day at least 9 days in the last 10 days, disable alias""" # """if an alias has bounces every day at least 9 days in the last 10 days, disable alias"""
user = login(flask_client) # user = login(flask_client)
alias = Alias.create_new_random(user) # alias = Alias.create_new_random(user)
db.session.commit() # db.session.commit()
#
assert not should_disable(alias) # assert not should_disable(alias)
#
# create a lot of bounce on this alias # # create a lot of bounce on this alias
contact = Contact.create( # contact = Contact.create(
user_id=user.id, # user_id=user.id,
alias_id=alias.id, # alias_id=alias.id,
website_email="contact@example.com", # website_email="contact@example.com",
reply_email="rep@sl.local", # reply_email="rep@sl.local",
commit=True, # commit=True,
) # )
for i in range(9): # for i in range(9):
EmailLog.create( # EmailLog.create(
user_id=user.id, # user_id=user.id,
contact_id=contact.id, # contact_id=contact.id,
alias_id=contact.alias_id, # alias_id=contact.alias_id,
commit=True, # commit=True,
bounced=True, # bounced=True,
created_at=arrow.now().shift(days=-i), # created_at=arrow.now().shift(days=-i),
) # )
#
assert should_disable(alias) # assert should_disable(alias)
def test_should_disable_bounces_account(flask_client): # def test_should_disable_bounces_account(flask_client):
"""if an account has more than 10 bounces every day for at least 5 days in the last 10 days, disable alias""" # """if an account has more than 10 bounces every day for at least 5 days in the last 10 days, disable alias"""
user = login(flask_client) # user = login(flask_client)
alias = Alias.create_new_random(user) # alias = Alias.create_new_random(user)
#
db.session.commit() # db.session.commit()
#
# create a lot of bounces on alias # # create a lot of bounces on alias
contact = Contact.create( # contact = Contact.create(
user_id=user.id, # user_id=user.id,
alias_id=alias.id, # alias_id=alias.id,
website_email="contact@example.com", # website_email="contact@example.com",
reply_email="rep@sl.local", # reply_email="rep@sl.local",
commit=True, # commit=True,
) # )
#
for day in range(6): # for day in range(6):
for _ in range(10): # for _ in range(10):
EmailLog.create( # EmailLog.create(
user_id=user.id, # user_id=user.id,
contact_id=contact.id, # contact_id=contact.id,
alias_id=contact.alias_id, # alias_id=contact.alias_id,
commit=True, # commit=True,
bounced=True, # bounced=True,
created_at=arrow.now().shift(days=-day), # created_at=arrow.now().shift(days=-day),
) # )
#
alias2 = Alias.create_new_random(user) # alias2 = Alias.create_new_random(user)
assert should_disable(alias2) # assert should_disable(alias2)
def test_should_disable_bounce_consecutive_days(flask_client): # def test_should_disable_bounce_consecutive_days(flask_client):
user = login(flask_client) # user = login(flask_client)
alias = Alias.create_new_random(user) # alias = Alias.create_new_random(user)
db.session.commit() # db.session.commit()
#
contact = Contact.create( # contact = Contact.create(
user_id=user.id, # user_id=user.id,
alias_id=alias.id, # alias_id=alias.id,
website_email="contact@example.com", # website_email="contact@example.com",
reply_email="rep@sl.local", # reply_email="rep@sl.local",
commit=True, # commit=True,
) # )
#
# create 6 bounce on this alias in the last 24h: alias is not disabled # # create 6 bounce on this alias in the last 24h: alias is not disabled
for _ in range(6): # for _ in range(6):
EmailLog.create( # EmailLog.create(
user_id=user.id, # user_id=user.id,
contact_id=contact.id, # contact_id=contact.id,
alias_id=contact.alias_id, # alias_id=contact.alias_id,
commit=True, # commit=True,
bounced=True, # bounced=True,
) # )
assert not should_disable(alias) # assert not should_disable(alias)
#
# create 2 bounces in the last 7 days: alias should be disabled # # create 2 bounces in the last 7 days: alias should be disabled
for _ in range(2): # for _ in range(2):
EmailLog.create( # EmailLog.create(
user_id=user.id, # user_id=user.id,
contact_id=contact.id, # contact_id=contact.id,
alias_id=contact.alias_id, # alias_id=contact.alias_id,
commit=True, # commit=True,
bounced=True, # bounced=True,
created_at=arrow.now().shift(days=-3), # created_at=arrow.now().shift(days=-3),
) # )
assert should_disable(alias) # assert should_disable(alias)
def test_parse_id_from_bounce(): def test_parse_id_from_bounce():