Mirror
/
app-MAIL-temp
mirror of https://github.com/simple-login/app.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cron, init app, job runner: wrap in an app context to benefit from app setup like database cleanup, sentry integration, etc

This commit is contained in:
Son 2021-10-26 10:52:28 +02:00
parent a0165d6381
commit a99ac24b72
5 changed files with 147 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
cron.py
View File

@ -59,6 +59,7 @@ from app.models import (
HibpNotifiedAlias, HibpNotifiedAlias,
) )
from app.utils import sanitize_email from app.utils import sanitize_email
from server import create_light_app
def notify_trial_end(): def notify_trial_end():
@ -895,37 +896,38 @@ if __name__ == "__main__":
], ],
) )
args = parser.parse_args() args = parser.parse_args()
# wrap in an app context to benefit from app setup like database cleanup, sentry integration, etc
if args.job == "stats": with create_light_app().app_context():
LOG.d("Compute Stats") if args.job == "stats":
stats() LOG.d("Compute Stats")
elif args.job == "notify_trial_end": stats()
LOG.d("Notify users with trial ending soon") elif args.job == "notify_trial_end":
notify_trial_end() LOG.d("Notify users with trial ending soon")
elif args.job == "notify_manual_subscription_end": notify_trial_end()
LOG.d("Notify users with manual subscription ending soon") elif args.job == "notify_manual_subscription_end":
notify_manual_sub_end() LOG.d("Notify users with manual subscription ending soon")
elif args.job == "notify_premium_end": notify_manual_sub_end()
LOG.d("Notify users with premium ending soon") elif args.job == "notify_premium_end":
notify_premium_end() LOG.d("Notify users with premium ending soon")
elif args.job == "delete_logs": notify_premium_end()
LOG.d("Deleted Logs") elif args.job == "delete_logs":
delete_logs() LOG.d("Deleted Logs")
elif args.job == "poll_apple_subscription": delete_logs()
LOG.d("Poll Apple Subscriptions") elif args.job == "poll_apple_subscription":
poll_apple_subscription() LOG.d("Poll Apple Subscriptions")
elif args.job == "sanity_check": poll_apple_subscription()
LOG.d("Check data consistency") elif args.job == "sanity_check":
sanity_check() LOG.d("Check data consistency")
elif args.job == "delete_old_monitoring": sanity_check()
LOG.d("Delete old monitoring records") elif args.job == "delete_old_monitoring":
delete_old_monitoring() LOG.d("Delete old monitoring records")
elif args.job == "check_custom_domain": delete_old_monitoring()
LOG.d("Check custom domain") elif args.job == "check_custom_domain":
check_custom_domain() LOG.d("Check custom domain")
elif args.job == "check_hibp": check_custom_domain()
LOG.d("Check HIBP") elif args.job == "check_hibp":
asyncio.run(check_hibp()) LOG.d("Check HIBP")
elif args.job == "notify_hibp": asyncio.run(check_hibp())
LOG.d("Notify users about HIBP breaches") elif args.job == "notify_hibp":
notify_hibp() LOG.d("Notify users about HIBP breaches")
notify_hibp()

16
email_handler.py
View File

@ -150,19 +150,6 @@ if NEWRELIC_CONFIG_PATH:
newrelic_app = newrelic.agent.register_application() newrelic_app = newrelic.agent.register_application()
# fix the database connection leak issue
# use this method instead of create_app
def new_app():
app = create_light_app()
@app.teardown_appcontext
def shutdown_session(response_or_exc):
# same as shutdown_session() in flask-sqlalchemy but this is not enough
Session.remove()
return app
def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Contact: def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Contact:
""" """
contact_from_header is the RFC 2047 format FROM header contact_from_header is the RFC 2047 format FROM header
@ -2059,8 +2046,7 @@ class MailHandler:
envelope.rcpt_tos, envelope.rcpt_tos,
) )
app = new_app() with create_light_app().app_context():
with app.app_context():
ret = handle(envelope) ret = handle(envelope)
elapsed = time.time() - start elapsed = time.time() - start
LOG.i( LOG.i(

7
init_app.py
View File

@ -3,6 +3,7 @@ from app.db import Session
from app.log import LOG from app.log import LOG
from app.models import Mailbox, Contact, SLDomain from app.models import Mailbox, Contact, SLDomain
from app.pgp_utils import load_public_key from app.pgp_utils import load_public_key
from server import create_light_app
def load_pgp_public_keys(): def load_pgp_public_keys():
@ -50,5 +51,7 @@ def add_sl_domains():
if __name__ == "__main__": if __name__ == "__main__":
load_pgp_public_keys() # wrap in an app context to benefit from app setup like database cleanup, sentry integration, etc
add_sl_domains() with create_light_app().app_context():
load_pgp_public_keys()
add_sl_domains()

213
job_runner.py
View File

@ -26,19 +26,6 @@ from app.models import User, Job, BatchImport, Mailbox, CustomDomain
from server import create_light_app from server import create_light_app
# fix the database connection leak issue
# use this method instead of create_app
def new_app():
app = create_light_app()
@app.teardown_appcontext
def shutdown_session(response_or_exc):
# same as shutdown_session() in flask-sqlalchemy but this is not enough
Session.remove()
return app
def onboarding_send_from_alias(user): def onboarding_send_from_alias(user):
to_email, unsubscribe_link, via_email = user.get_communication_email() to_email, unsubscribe_link, via_email = user.get_communication_email()
if not to_email: if not to_email:
@ -101,116 +88,118 @@ def onboarding_mailbox(user):
if __name__ == "__main__": if __name__ == "__main__":
while True: while True:
# run a job 1h earlier or later is not a big deal ... # wrap in an app context to benefit from app setup like database cleanup, sentry integration, etc
min_dt = arrow.now().shift(hours=-1) with create_light_app().app_context():
max_dt = arrow.now().shift(hours=1) # run a job 1h earlier or later is not a big deal ...
min_dt = arrow.now().shift(hours=-1)
max_dt = arrow.now().shift(hours=1)
for job in Job.filter( for job in Job.filter(
Job.taken.is_(False), Job.run_at > min_dt, Job.run_at <= max_dt Job.taken.is_(False), Job.run_at > min_dt, Job.run_at <= max_dt
).all(): ).all():
LOG.d("Take job %s", job) LOG.d("Take job %s", job)
# mark the job as taken, whether it will be executed successfully or not # mark the job as taken, whether it will be executed successfully or not
job.taken = True job.taken = True
Session.commit()
if job.name == JOB_ONBOARDING_1:
user_id = job.payload.get("user_id")
user = User.get(user_id)
# user might delete their account in the meantime
# or disable the notification
if user and user.notification and user.activated:
LOG.d("send onboarding send-from-alias email to user %s", user)
onboarding_send_from_alias(user)
elif job.name == JOB_ONBOARDING_2:
user_id = job.payload.get("user_id")
user = User.get(user_id)
# user might delete their account in the meantime
# or disable the notification
if user and user.notification and user.activated:
LOG.d("send onboarding mailbox email to user %s", user)
onboarding_mailbox(user)
elif job.name == JOB_ONBOARDING_4:
user_id = job.payload.get("user_id")
user = User.get(user_id)
# user might delete their account in the meantime
# or disable the notification
if user and user.notification and user.activated:
LOG.d("send onboarding pgp email to user %s", user)
onboarding_pgp(user)
elif job.name == JOB_BATCH_IMPORT:
batch_import_id = job.payload.get("batch_import_id")
batch_import = BatchImport.get(batch_import_id)
handle_batch_import(batch_import)
elif job.name == JOB_DELETE_ACCOUNT:
user_id = job.payload.get("user_id")
user = User.get(user_id)
if not user:
LOG.i("No user found for %s", user_id)
continue
user_email = user.email
LOG.w("Delete user %s", user)
User.delete(user.id)
Session.commit() Session.commit()
send_email( if job.name == JOB_ONBOARDING_1:
user_email, user_id = job.payload.get("user_id")
"Your SimpleLogin account has been deleted", user = User.get(user_id)
render("transactional/account-delete.txt"),
render("transactional/account-delete.html"),
)
elif job.name == JOB_DELETE_MAILBOX:
mailbox_id = job.payload.get("mailbox_id")
mailbox = Mailbox.get(mailbox_id)
if not mailbox:
continue
mailbox_email = mailbox.email # user might delete their account in the meantime
user = mailbox.user # or disable the notification
if user and user.notification and user.activated:
LOG.d("send onboarding send-from-alias email to user %s", user)
onboarding_send_from_alias(user)
elif job.name == JOB_ONBOARDING_2:
user_id = job.payload.get("user_id")
user = User.get(user_id)
Mailbox.delete(mailbox_id) # user might delete their account in the meantime
Session.commit() # or disable the notification
LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email) if user and user.notification and user.activated:
LOG.d("send onboarding mailbox email to user %s", user)
onboarding_mailbox(user)
elif job.name == JOB_ONBOARDING_4:
user_id = job.payload.get("user_id")
user = User.get(user_id)
send_email( # user might delete their account in the meantime
user.email, # or disable the notification
f"Your mailbox {mailbox_email} has been deleted", if user and user.notification and user.activated:
f"""Mailbox {mailbox_email} along with its aliases are deleted successfully. LOG.d("send onboarding pgp email to user %s", user)
Regards, onboarding_pgp(user)
SimpleLogin team.
""",
)
elif job.name == JOB_DELETE_DOMAIN: elif job.name == JOB_BATCH_IMPORT:
custom_domain_id = job.payload.get("custom_domain_id") batch_import_id = job.payload.get("batch_import_id")
custom_domain = CustomDomain.get(custom_domain_id) batch_import = BatchImport.get(batch_import_id)
if not custom_domain: handle_batch_import(batch_import)
continue elif job.name == JOB_DELETE_ACCOUNT:
user_id = job.payload.get("user_id")
user = User.get(user_id)
domain_name = custom_domain.domain if not user:
user = custom_domain.user LOG.i("No user found for %s", user_id)
continue
CustomDomain.delete(custom_domain.id) user_email = user.email
Session.commit() LOG.w("Delete user %s", user)
User.delete(user.id)
Session.commit()
LOG.d("Domain %s deleted", domain_name) send_email(
user_email,
"Your SimpleLogin account has been deleted",
render("transactional/account-delete.txt"),
render("transactional/account-delete.html"),
)
elif job.name == JOB_DELETE_MAILBOX:
mailbox_id = job.payload.get("mailbox_id")
mailbox = Mailbox.get(mailbox_id)
if not mailbox:
continue
send_email( mailbox_email = mailbox.email
user.email, user = mailbox.user
f"Your domain {domain_name} has been deleted",
f"""Domain {domain_name} along with its aliases are deleted successfully.
Regards,
SimpleLogin team.
""",
)
else: Mailbox.delete(mailbox_id)
LOG.e("Unknown job name %s", job.name) Session.commit()
LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email)
time.sleep(10) send_email(
user.email,
f"Your mailbox {mailbox_email} has been deleted",
f"""Mailbox {mailbox_email} along with its aliases are deleted successfully.
Regards,
SimpleLogin team.
""",
)
elif job.name == JOB_DELETE_DOMAIN:
custom_domain_id = job.payload.get("custom_domain_id")
custom_domain = CustomDomain.get(custom_domain_id)
if not custom_domain:
continue
domain_name = custom_domain.domain
user = custom_domain.user
CustomDomain.delete(custom_domain.id)
Session.commit()
LOG.d("Domain %s deleted", domain_name)
send_email(
user.email,
f"Your domain {domain_name} has been deleted",
f"""Domain {domain_name} along with its aliases are deleted successfully.
Regards,
SimpleLogin team.
""",
)
else:
LOG.e("Unknown job name %s", job.name)
time.sleep(10)

4
server.py
View File

@ -131,6 +131,10 @@ def create_light_app() -> Flask:
app.config["SQLALCHEMY_DATABASE_URI"] = DB_URI app.config["SQLALCHEMY_DATABASE_URI"] = DB_URI
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
@app.teardown_appcontext
def shutdown_session(response_or_exc):
Session.remove()
return app return app