import argparse
from dataclasses import dataclass
from time import sleep
import arrow
from arrow import Arrow
from app import s3
from app.api.views.apple import verify_receipt
from app.config import (
IGNORED_EMAILS,
ADMIN_EMAIL,
MACAPP_APPLE_API_SECRET,
APPLE_API_SECRET,
)
from app.email_utils import (
send_email,
send_trial_end_soon_email,
render,
email_domain_can_be_used_as_mailbox,
)
from app.extensions import db
from app.log import LOG
from app.models import (
Subscription,
User,
Alias,
EmailLog,
CustomDomain,
Client,
ManualSubscription,
RefusedEmail,
AppleSubscription,
Mailbox,
)
from server import create_app
def notify_trial_end():
for user in User.query.filter(
User.activated == True, User.trial_end.isnot(None), User.lifetime == False
).all():
if user.in_trial() and arrow.now().shift(
days=3
) > user.trial_end >= arrow.now().shift(days=2):
LOG.d("Send trial end email to user %s", user)
send_trial_end_soon_email(user)
def delete_refused_emails():
for refused_email in RefusedEmail.query.filter(RefusedEmail.deleted == False).all():
if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now():
LOG.d("Delete refused email %s", refused_email)
if refused_email.path:
s3.delete(refused_email.path)
s3.delete(refused_email.full_report_path)
# do not set path and full_report_path to null
# so we can check later that the files are indeed deleted
refused_email.deleted = True
db.session.commit()
LOG.d("Finish delete_refused_emails")
def notify_premium_end():
"""sent to user who has canceled their subscription and who has their subscription ending soon"""
for sub in Subscription.query.filter(Subscription.cancelled == True).all():
if (
arrow.now().shift(days=3).date()
> sub.next_bill_date
>= arrow.now().shift(days=2).date()
):
user = sub.user
LOG.d(f"Send subscription ending soon email to user {user}")
send_email(
user.email,
f"Your subscription will end soon {user.name}",
render(
"transactional/subscription-end.txt",
user=user,
next_bill_date=sub.next_bill_date.strftime("%Y-%m-%d"),
),
render(
"transactional/subscription-end.html",
user=user,
next_bill_date=sub.next_bill_date.strftime("%Y-%m-%d"),
),
)
def notify_manual_sub_end():
for manual_sub in ManualSubscription.query.all():
need_reminder = False
if arrow.now().shift(days=14) > manual_sub.end_at > arrow.now().shift(days=13):
need_reminder = True
elif arrow.now().shift(days=4) > manual_sub.end_at > arrow.now().shift(days=3):
need_reminder = True
if need_reminder:
user = manual_sub.user
LOG.debug("Remind user %s that their manual sub is ending soon", user)
send_email(
user.email,
f"Your trial will end soon {user.name}",
render(
"transactional/manual-subscription-end.txt",
name=user.name,
user=user,
manual_sub=manual_sub,
),
render(
"transactional/manual-subscription-end.html",
name=user.name,
user=user,
manual_sub=manual_sub,
),
)
def poll_apple_subscription():
"""Poll Apple API to update AppleSubscription"""
# todo: only near the end of the subscription
for apple_sub in AppleSubscription.query.all():
user = apple_sub.user
verify_receipt(apple_sub.receipt_data, user, APPLE_API_SECRET)
verify_receipt(apple_sub.receipt_data, user, MACAPP_APPLE_API_SECRET)
LOG.d("Finish poll_apple_subscription")
@dataclass
class Stats:
nb_user: int
nb_alias: int
nb_forward: int
nb_block: int
nb_reply: int
nb_bounced: int
nb_spam: int
nb_custom_domain: int
nb_app: int
nb_premium: int
nb_apple_premium: int
def stats_before(moment: Arrow) -> Stats:
"""return the stats before a specific moment, ignoring all stats come from users in IGNORED_EMAILS
"""
# nb user
q = User.query
for ie in IGNORED_EMAILS:
q = q.filter(~User.email.contains(ie), User.created_at < moment)
nb_user = q.count()
LOG.d("total number user %s", nb_user)
# nb alias
q = db.session.query(Alias, User).filter(
Alias.user_id == User.id, Alias.created_at < moment
)
for ie in IGNORED_EMAILS:
q = q.filter(~User.email.contains(ie))
nb_alias = q.count()
LOG.d("total number alias %s", nb_alias)
# email log stats
q = (
db.session.query(EmailLog)
.join(User, EmailLog.user_id == User.id)
.filter(EmailLog.created_at < moment,)
)
for ie in IGNORED_EMAILS:
q = q.filter(~User.email.contains(ie))
nb_spam = nb_bounced = nb_forward = nb_block = nb_reply = 0
for email_log in q.yield_per(500):
if email_log.bounced:
nb_bounced += 1
elif email_log.is_spam:
nb_spam += 1
elif email_log.is_reply:
nb_reply += 1
elif email_log.blocked:
nb_block += 1
else:
nb_forward += 1
LOG.d(
"nb_forward %s, nb_block %s, nb_reply %s, nb_bounced %s, nb_spam %s",
nb_forward,
nb_block,
nb_reply,
nb_bounced,
nb_spam,
)
nb_premium = Subscription.query.filter(
Subscription.created_at < moment, Subscription.cancelled == False
).count()
nb_apple_premium = AppleSubscription.query.filter(
AppleSubscription.created_at < moment
).count()
nb_custom_domain = CustomDomain.query.filter(
CustomDomain.created_at < moment
).count()
nb_app = Client.query.filter(Client.created_at < moment).count()
data = locals()
# to keep only Stats field
data = {
k: v
for (k, v) in data.items()
if k in vars(Stats)["__dataclass_fields__"].keys()
}
return Stats(**data)
def increase_percent(old, new) -> str:
if old == 0:
return "N/A"
increase = (new - old) / old * 100
return f"{increase:.1f}%"
def stats():
"""send admin stats everyday"""
if not ADMIN_EMAIL:
# nothing to do
return
stats_today = stats_before(arrow.now())
stats_yesterday = stats_before(arrow.now().shift(days=-1))
nb_user_increase = increase_percent(stats_yesterday.nb_user, stats_today.nb_user)
nb_alias_increase = increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)
nb_forward_increase = increase_percent(
stats_yesterday.nb_forward, stats_today.nb_forward
)
today = arrow.now().format()
send_email(
ADMIN_EMAIL,
subject=f"SimpleLogin Stats for {today}, {nb_user_increase} users, {nb_alias_increase} aliases, {nb_forward_increase} forwards",
plaintext="",
html=f"""
Stats for {today}
nb_user: {stats_today.nb_user} - {increase_percent(stats_yesterday.nb_user, stats_today.nb_user)}
nb_premium: {stats_today.nb_premium} - {increase_percent(stats_yesterday.nb_premium, stats_today.nb_premium)}
nb_apple_premium: {stats_today.nb_apple_premium} - {increase_percent(stats_yesterday.nb_apple_premium, stats_today.nb_apple_premium)}
nb_alias: {stats_today.nb_alias} - {increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)}
nb_forward: {stats_today.nb_forward} - {increase_percent(stats_yesterday.nb_forward, stats_today.nb_forward)}
nb_reply: {stats_today.nb_reply} - {increase_percent(stats_yesterday.nb_reply, stats_today.nb_reply)}
nb_block: {stats_today.nb_block} - {increase_percent(stats_yesterday.nb_block, stats_today.nb_block)}
nb_bounced: {stats_today.nb_bounced} - {increase_percent(stats_yesterday.nb_bounced, stats_today.nb_bounced)}
nb_spam: {stats_today.nb_spam} - {increase_percent(stats_yesterday.nb_spam, stats_today.nb_spam)}
nb_custom_domain: {stats_today.nb_custom_domain} - {increase_percent(stats_yesterday.nb_custom_domain, stats_today.nb_custom_domain)}
nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)}
""",
)
def sanity_check():
"""
#TODO: investigate why DNS sometimes not working
Different sanity checks
- detect if there's mailbox that's using a invalid domain
"""
for mailbox in Mailbox.filter_by(verified=True).all():
# hack to not query DNS too often
sleep(1)
if not email_domain_can_be_used_as_mailbox(mailbox.email):
mailbox.nb_failed_checks += 1
# alert if too much fail
if mailbox.nb_failed_checks > 10:
log_func = LOG.exception
else:
log_func = LOG.warning
log_func(
"issue with mailbox %s domain. #alias %s, nb email log %s",
mailbox,
mailbox.nb_alias(),
mailbox.nb_email_log(),
)
else: # reset nb check
mailbox.nb_failed_checks = 0
db.session.commit()
for user in User.filter_by(activated=True).all():
if user.email.lower() != user.email:
LOG.exception("%s does not have lowercase email", user)
for mailbox in Mailbox.filter_by(verified=True).all():
if mailbox.email.lower() != mailbox.email:
LOG.exception("%s does not have lowercase email", mailbox)
LOG.d("Finish sanity check")
if __name__ == "__main__":
LOG.d("Start running cronjob")
parser = argparse.ArgumentParser()
parser.add_argument(
"-j",
"--job",
help="Choose a cron job to run",
type=str,
choices=[
"stats",
"notify_trial_end",
"notify_manual_subscription_end",
"notify_premium_end",
"delete_refused_emails",
"poll_apple_subscription",
"sanity_check",
],
)
args = parser.parse_args()
app = create_app()
with app.app_context():
if args.job == "stats":
LOG.d("Compute Stats")
stats()
elif args.job == "notify_trial_end":
LOG.d("Notify users with trial ending soon")
notify_trial_end()
elif args.job == "notify_manual_subscription_end":
LOG.d("Notify users with manual subscription ending soon")
notify_manual_sub_end()
elif args.job == "notify_premium_end":
LOG.d("Notify users with premium ending soon")
notify_premium_end()
elif args.job == "delete_refused_emails":
LOG.d("Deleted refused emails")
delete_refused_emails()
elif args.job == "poll_apple_subscription":
LOG.d("Poll Apple Subscriptions")
poll_apple_subscription()
elif args.job == "sanity_check":
LOG.d("Check data consistency")
sanity_check()