Mirror
/
app-MAIL-temp
mirror of https://github.com/simple-login/app.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'simple-login:master' into master

This commit is contained in:
Job 2021-10-15 14:09:52 +02:00 committed by GitHub
parent 4e5ca3b30b 7bfdb821af
commit 742bfd6815
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 551 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CONTRIBUTING.md
View File

@ -76,7 +76,7 @@ docker run -e POSTGRES_PASSWORD=mypassword -e POSTGRES_USER=myuser -e POSTGRES_D
To run the server:
```
flask db upgrade && flask dummy-data && python3 server.py
alembic upgrade head && flask dummy-data && python3 server.py
```
then open http://localhost:7777, you should be able to login with `john@wick.com / password` account.

15
README.md
View File

@ -473,6 +473,21 @@ docker run -d \
simplelogin/app:3.4.0 python email_handler.py
```
And finally the `job runner`
```bash
docker run -d \
--name sl-job-runner \
-v $(pwd)/sl:/sl \
-v $(pwd)/sl/upload:/code/static/upload \
-v $(pwd)/simplelogin.env:/code/.env \
-v $(pwd)/dkim.key:/dkim.key \
-v $(pwd)/dkim.pub.key:/dkim.pub.key \
--restart always \
--network="sl-network" \
simplelogin/app:3.4.0 python job_runner.py
```
### Nginx
Install Nginx and make sure to replace `mydomain.com` by your domain

83
alembic.ini Normal file
View File

@ -0,0 +1,83 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = migrations
# template used to generate migration files
file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s
# timezone to use when rendering the date
# within the migration file as well as the filename.
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; this defaults
# to alembic/versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat alembic/versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks=black
# black.type=console_scripts
# black.entrypoint=black
# black.options=-l 79
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

15
app/api/views/mailbox.py
View File

@ -1,10 +1,12 @@
from smtplib import SMTPRecipientsRefused
import arrow
from flask import g
from flask import jsonify
from flask import request
from app.api.base import api_bp, require_api_auth
from app.config import JOB_DELETE_MAILBOX
from app.dashboard.views.mailbox import send_verification_email
from app.dashboard.views.mailbox_detail import verify_mailbox_change
from app.db import Session
@ -13,7 +15,8 @@ from app.email_utils import (
email_can_be_used_as_mailbox,
is_valid_email,
)
from app.models import Mailbox
from app.log import LOG
from app.models import Mailbox, Job
from app.utils import sanitize_email
@ -88,8 +91,14 @@ def delete_mailbox(mailbox_id):
if mailbox.id == user.default_mailbox_id:
return jsonify(error="You cannot delete the default mailbox"), 400
Mailbox.delete(mailbox_id)
Session.commit()
# Schedule delete account job
LOG.w("schedule delete mailbox job for %s", mailbox)
Job.create(
name=JOB_DELETE_MAILBOX,
payload={"mailbox_id": mailbox.id},
run_at=arrow.now(),
commit=True,
)
return jsonify(deleted=True), 200

2
app/config.py
View File

@ -255,6 +255,8 @@ JOB_ONBOARDING_3 = "onboarding-3"
JOB_ONBOARDING_4 = "onboarding-4"
JOB_BATCH_IMPORT = "batch-import"
JOB_DELETE_ACCOUNT = "delete-account"
JOB_DELETE_MAILBOX = "delete-mailbox"
JOB_DELETE_DOMAIN = "delete-domain"
# for pagination
PAGE_LIMIT = 20

8
app/dashboard/views/custom_domain.py
View File

@ -43,13 +43,19 @@ def custom_domain():
if SLDomain.get_by(domain=new_domain):
flash("A custom domain cannot be a built-in domain.", "error")
elif CustomDomain.get_by(domain=new_domain):
flash(f"{new_domain} already used", "warning")
flash(f"{new_domain} already used", "error")
elif get_email_domain_part(current_user.email) == new_domain:
flash(
"You cannot add a domain that you are currently using for your personal email. "
"Please change your personal email to your real email",
"error",
)
elif Mailbox.filter(
Mailbox.verified.is_(True), Mailbox.email.endswith(f"@{new_domain}")
).first():
flash(
f"{new_domain} already used in a SimpleLogin mailbox", "error"
)
else:
new_custom_domain = CustomDomain.create(
domain=new_domain, user_id=current_user.id

45
app/dashboard/views/domain_detail.py
View File

@ -1,12 +1,11 @@
from threading import Thread
import arrow
import re2 as re
from flask import render_template, request, redirect, url_for, flash
from flask_login import login_required, current_user
from flask_wtf import FlaskForm
from wtforms import StringField, validators, IntegerField
from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN
from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN, JOB_DELETE_DOMAIN
from app.dashboard.base import dashboard_bp
from app.db import Session
from app.dns_utils import (
@ -15,7 +14,6 @@ from app.dns_utils import (
get_txt_record,
get_cname_record,
)
from app.email_utils import send_email
from app.log import LOG
from app.models import (
CustomDomain,
@ -25,6 +23,7 @@ from app.models import (
DomainMailbox,
AutoCreateRule,
AutoCreateRuleMailbox,
Job,
)
from app.utils import random_string
@ -276,7 +275,16 @@ def domain_detail(custom_domain_id):
elif request.form.get("form-name") == "delete":
name = custom_domain.domain
LOG.d("Schedule deleting %s", custom_domain)
Thread(target=delete_domain, args=(custom_domain_id,)).start()
# Schedule delete domain job
LOG.w("schedule delete domain job for %s", custom_domain)
Job.create(
name=JOB_DELETE_DOMAIN,
payload={"custom_domain_id": custom_domain.id},
run_at=arrow.now(),
commit=True,
)
flash(
f"{name} scheduled for deletion."
f"You will receive a confirmation email when the deletion is finished",
@ -290,33 +298,6 @@ def domain_detail(custom_domain_id):
return render_template("dashboard/domain_detail/info.html", **locals())
def delete_domain(custom_domain_id: int):
from server import create_light_app
with create_light_app().app_context():
custom_domain = CustomDomain.get(custom_domain_id)
if not custom_domain:
return
domain_name = custom_domain.domain
user = custom_domain.user
CustomDomain.delete(custom_domain.id)
Session.commit()
LOG.d("Domain %s deleted", domain_name)
send_email(
user.email,
f"Your domain {domain_name} has been deleted",
f"""Domain {domain_name} along with its aliases are deleted successfully.
Regards,
SimpleLogin team.
""",
)
@dashboard_bp.route("/domains/<int:custom_domain_id>/trash", methods=["GET", "POST"])
@login_required
def domain_detail_trash(custom_domain_id):

18
app/dashboard/views/mailbox.py
View File

@ -1,5 +1,4 @@
from threading import Thread
import arrow
from flask import render_template, request, redirect, url_for, flash
from flask_login import login_required, current_user
from flask_wtf import FlaskForm
@ -7,7 +6,7 @@ from itsdangerous import Signer
from wtforms import validators
from wtforms.fields.html5 import EmailField
from app.config import MAILBOX_SECRET, URL
from app.config import MAILBOX_SECRET, URL, JOB_DELETE_MAILBOX
from app.dashboard.base import dashboard_bp
from app.db import Session
from app.email_utils import (
@ -18,7 +17,7 @@ from app.email_utils import (
is_valid_email,
)
from app.log import LOG
from app.models import Mailbox
from app.models import Mailbox, Job
class NewMailboxForm(FlaskForm):
@ -51,8 +50,15 @@ def mailbox_route():
flash("You cannot delete default mailbox", "error")
return redirect(url_for("dashboard.mailbox_route"))
LOG.d("Schedule deleting %s", mailbox)
Thread(target=delete_mailbox, args=(mailbox.id,)).start()
# Schedule delete account job
LOG.w("schedule delete mailbox job for %s", mailbox)
Job.create(
name=JOB_DELETE_MAILBOX,
payload={"mailbox_id": mailbox.id},
run_at=arrow.now(),
commit=True,
)
flash(
f"Mailbox {mailbox.email} scheduled for deletion."
f"You will receive a confirmation email when the deletion is finished",

3
app/email/rate_limit.py
View File

@ -93,6 +93,9 @@ def rate_limited_reply_phase(reply_email: str) -> bool:
def rate_limited(mail_from: str, rcpt_tos: [str]) -> bool:
# todo: re-enable rate limiting
return False
for rcpt_to in rcpt_tos:
if is_reply_email(rcpt_to):
if rate_limited_reply_phase(rcpt_to):

45
app/email_utils.py
View File

@ -598,22 +598,53 @@ def mailbox_already_used(email: str, user) -> bool:
return False
def get_orig_message_from_bounce(msg: Message) -> Message:
def get_orig_message_from_bounce(bounce_report: Message) -> Optional[Message]:
"""parse the original email from Bounce"""
i = 0
for part in msg.walk():
for part in bounce_report.walk():
i += 1
# the original message is the 4th part
# 1st part is the root part, multipart/report
# 2nd is text/plain, Postfix log
# 1st part is the container (bounce report)
# 2nd part is the report from our own Postfix
# 3rd is report from other mailbox
# 4th is the container of the original message
# ...
# 7th is original message
if i == 7:
return part
def get_orig_message_from_hotmail_complaint(msg: Message) -> Message:
def get_mailbox_bounce_info(bounce_report: Message) -> Optional[Message]:
"""
Return the bounce info from the bounce report
An example of bounce info:
Final-Recipient: rfc822; not-existing@gmail.com
Original-Recipient: rfc822;not-existing@gmail.com
Action: failed
Status: 5.1.1
Remote-MTA: dns; gmail-smtp-in.l.google.com
Diagnostic-Code: smtp;
550-5.1.1 The email account that you tried to reach does
not exist. Please try 550-5.1.1 double-checking the recipient's email
address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
"""
i = 0
for part in bounce_report.walk():
i += 1
# 1st part is the container (bounce report)
# 2nd part is the report from our own Postfix
# 3rd is report from other mailbox
# 4th is the container of the original message
# 5th is a child of 3rd that contains more info about the bounce
if i == 5:
return part
def get_orig_message_from_hotmail_complaint(msg: Message) -> Optional[Message]:
i = 0
for part in msg.walk():
i += 1
@ -625,7 +656,7 @@ def get_orig_message_from_hotmail_complaint(msg: Message) -> Message:
return part
def get_orig_message_from_yahoo_complaint(msg: Message) -> Message:
def get_orig_message_from_yahoo_complaint(msg: Message) -> Optional[Message]:
i = 0
for part in msg.walk():
i += 1

5
app/models.py
View File

@ -2503,6 +2503,8 @@ class Metric2(Base, ModelMixin):
nb_block_last_24h = sa.Column(sa.Float, nullable=True)
nb_reply_last_24h = sa.Column(sa.Float, nullable=True)
nb_bounced_last_24h = sa.Column(sa.Float, nullable=True)
# includes bounces for both forwarding and transactional email
nb_total_bounced_last_24h = sa.Column(sa.Float, nullable=True)
nb_verified_custom_domain = sa.Column(sa.Float, nullable=True)
@ -2514,6 +2516,7 @@ class Bounce(Base, ModelMixin):
__tablename__ = "bounce"
email = sa.Column(sa.String(256), nullable=False, index=True)
info = sa.Column(sa.Text, nullable=True)
class TransactionalEmail(Base, ModelMixin):
@ -2557,7 +2560,7 @@ class IgnoredEmail(Base, ModelMixin):
class IgnoreBounceSender(Base, ModelMixin):
"""Ignore sender that doesn't correctly handle bounces, for example noreply@github.com"""
__tablename__ = "ignored_bounce_sender"
__tablename__ = "ignore_bounce_sender"
mail_from = sa.Column(sa.String(512), nullable=False, unique=True)

2
cron.py
View File

@ -257,6 +257,7 @@ def compute_metric2() -> Metric2:
nb_bounced_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
.filter_by(bounced=True)
.count(),
nb_total_bounced_last_24h=Bounce.filter(Bounce.created_at > _24h_ago).count(),
nb_reply_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
.filter_by(is_reply=True)
.count(),
@ -408,6 +409,7 @@ nb_forward_last_24h: {stats_today.nb_forward_last_24h} - {increase_percent(stats
nb_reply_last_24h: {stats_today.nb_reply_last_24h} - {increase_percent(stats_yesterday.nb_reply_last_24h, stats_today.nb_reply_last_24h)} <br>
nb_block_last_24h: {stats_today.nb_block_last_24h} - {increase_percent(stats_yesterday.nb_block_last_24h, stats_today.nb_block_last_24h)} <br>
nb_bounced_last_24h: {stats_today.nb_bounced_last_24h} - {increase_percent(stats_yesterday.nb_bounced_last_24h, stats_today.nb_bounced_last_24h)} <br>
nb_total_bounced_last_24h: {stats_today.nb_total_bounced_last_24h} - {increase_percent(stats_yesterday.nb_total_bounced_last_24h, stats_today.nb_total_bounced_last_24h)} <br>
nb_custom_domain: {stats_today.nb_verified_custom_domain} - {increase_percent(stats_yesterday.nb_verified_custom_domain, stats_today.nb_verified_custom_domain)} <br>
nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)} <br>

15
docs/upgrade.md
View File

@ -126,7 +126,7 @@ for user in User.query.all():
sudo docker pull simplelogin/app:3.4.0
# Stop SimpleLogin containers
sudo docker stop sl-email sl-migration sl-app sl-db
sudo docker stop sl-email sl-migration sl-app sl-db sl-job-runner
# Make sure to remove these containers to avoid conflict
sudo docker rm -f sl-email sl-migration sl-app sl-db
@ -193,5 +193,18 @@ sudo docker run -d \
--restart always \
--network="sl-network" \
simplelogin/app:3.4.0 python email_handler.py
# Run the job runner
docker run -d \
--name sl-job-runner \
-v $(pwd)/sl:/sl \
-v $(pwd)/sl/upload:/code/static/upload \
-v $(pwd)/simplelogin.env:/code/.env \
-v $(pwd)/dkim.key:/dkim.key \
-v $(pwd)/dkim.pub.key:/dkim.pub.key \
--restart always \
--network="sl-network" \
simplelogin/app:3.4.0 python job_runner.py
```

127
email_handler.py
View File

@ -31,7 +31,6 @@ It should contain the following info:
"""
import argparse
import asyncio
import email
import os
import time
@ -124,6 +123,7 @@ from app.email_utils import (
get_orig_message_from_hotmail_complaint,
parse_full_address,
get_orig_message_from_yahoo_complaint,
get_mailbox_bounce_info,
)
from app.log import LOG, set_message_id
from app.models import (
@ -1179,7 +1179,12 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
LOG.e("Use %s default mailbox %s", alias, alias.mailbox)
mailbox = alias.mailbox
Bounce.create(email=mailbox.email, commit=True)
bounce_info = get_mailbox_bounce_info(msg)
if bounce_info:
Bounce.create(email=mailbox.email, info=bounce_info.as_string(), commit=True)
else:
LOG.w("cannot get bounce info")
Bounce.create(email=mailbox.email, commit=True)
LOG.d(
"Handle forward bounce %s -> %s -> %s. %s", contact, alias, mailbox, email_log
@ -1260,10 +1265,7 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
ignore_smtp_error=True,
)
else:
LOG.w(
"Disable alias %s now",
alias,
)
LOG.w("Disable alias %s %s. Last contact %s", alias, user, contact)
alias.enabled = False
Session.commit()
@ -1381,7 +1383,16 @@ def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog):
LOG.d("Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log)
Bounce.create(email=sanitize_email(contact.website_email), commit=True)
bounce_info = get_mailbox_bounce_info(msg)
if bounce_info:
Bounce.create(
email=sanitize_email(contact.website_email),
info=bounce_info.as_string(),
commit=True,
)
else:
LOG.w("cannot get bounce info")
Bounce.create(email=sanitize_email(contact.website_email), commit=True)
# Store the bounced email
# generate a name for the email
@ -1626,7 +1637,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
return status.E202
def handle_transactional_bounce(envelope: Envelope, rcpt_to):
def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to):
LOG.d("handle transactional bounce sent to %s", rcpt_to)
# parse the TransactionalEmail
@ -1636,7 +1647,14 @@ def handle_transactional_bounce(envelope: Envelope, rcpt_to):
# a transaction might have been deleted in delete_logs()
if transactional:
LOG.i("Create bounce for %s", transactional.email)
Bounce.create(email=transactional.email, commit=True)
bounce_info = get_mailbox_bounce_info(msg)
if bounce_info:
Bounce.create(
email=transactional.email, info=bounce_info.as_string(), commit=True
)
else:
LOG.w("cannot get bounce info")
Bounce.create(email=transactional.email, commit=True)
def handle_bounce(envelope, email_log: EmailLog, msg: Message) -> str:
@ -1712,7 +1730,7 @@ def should_ignore(mail_from: str, rcpt_tos: List[str]) -> bool:
return False
async def handle(envelope: Envelope) -> str:
def handle(envelope: Envelope) -> str:
"""Return SMTP status"""
# sanitize mail_from, rcpt_tos
@ -1726,7 +1744,11 @@ async def handle(envelope: Envelope) -> str:
if postfix_queue_id:
set_message_id(postfix_queue_id)
else:
LOG.d("Cannot parse Postfix queue ID from %s", msg[headers.RECEIVED])
LOG.d(
"Cannot parse Postfix queue ID from %s %s",
msg.get_all(headers.RECEIVED),
msg[headers.RECEIVED],
)
if should_ignore(mail_from, rcpt_tos):
LOG.w("Ignore email mail_from=%s rcpt_to=%s", mail_from, rcpt_tos)
@ -1774,7 +1796,7 @@ async def handle(envelope: Envelope) -> str:
and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
):
LOG.d("Handle email sent to sender from %s", mail_from)
handle_transactional_bounce(envelope, rcpt_tos[0])
handle_transactional_bounce(envelope, msg, rcpt_tos[0])
return status.E205
if (
@ -1858,37 +1880,24 @@ async def handle(envelope: Envelope) -> str:
return status.E523
if rate_limited(mail_from, rcpt_tos):
LOG.w(
"Rate Limiting applied for mail_from:%s rcpt_tos:%s, retry in 60s",
mail_from,
rcpt_tos,
)
# slow down the rate a bit
await asyncio.sleep(60)
LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos)
# rate limit is still applied
if rate_limited(mail_from, rcpt_tos):
LOG.w(
"Rate Limiting (no retry) applied for mail_from:%s rcpt_tos:%s",
mail_from,
rcpt_tos,
)
# add more logging info. TODO: remove
if len(rcpt_tos) == 1:
alias = Alias.get_by(email=rcpt_tos[0])
if alias:
LOG.w(
"total number email log on %s, %s is %s, %s",
alias,
alias.user,
EmailLog.filter(EmailLog.alias_id == alias.id).count(),
EmailLog.filter(EmailLog.user_id == alias.user_id).count(),
)
# add more logging info. TODO: remove
if len(rcpt_tos) == 1:
alias = Alias.get_by(email=rcpt_tos[0])
if alias:
LOG.w(
"total number email log on %s, %s is %s, %s",
alias,
alias.user,
EmailLog.filter(EmailLog.alias_id == alias.id).count(),
EmailLog.filter(EmailLog.user_id == alias.user_id).count(),
)
if should_ignore_bounce(envelope.mail_from):
return status.E207
else:
return status.E522
if should_ignore_bounce(envelope.mail_from):
return status.E207
else:
return status.E522
# Handle "out of office" auto notice. An automatic response is sent for every forwarded email
# todo: remove logging
@ -1948,7 +1957,7 @@ async def handle(envelope: Envelope) -> str:
class MailHandler:
async def handle_DATA(self, server, session, envelope: Envelope):
try:
ret = await self._handle(envelope)
ret = self._handle(envelope)
return ret
except Exception:
LOG.e(
@ -1959,7 +1968,7 @@ class MailHandler:
return status.E404
@newrelic.agent.background_task(application=newrelic_app)
async def _handle(self, envelope: Envelope):
def _handle(self, envelope: Envelope):
start = time.time()
# generate a different message_id to keep track of an email lifecycle
@ -1972,21 +1981,23 @@ class MailHandler:
envelope.rcpt_tos,
)
ret = await handle(envelope)
elapsed = time.time() - start
LOG.i(
"Finish mail from %s, rctp tos %s, takes %s seconds <<===",
envelope.mail_from,
envelope.rcpt_tos,
elapsed,
)
newrelic.agent.record_custom_metric(
"Custom/email_handler_time", elapsed, newrelic_app
)
newrelic.agent.record_custom_metric(
"Custom/number_incoming_email", 1, newrelic_app
)
return ret
app = new_app()
with app.app_context():
ret = handle(envelope)
elapsed = time.time() - start
LOG.i(
"Finish mail from %s, rctp tos %s, takes %s seconds <<===",
envelope.mail_from,
envelope.rcpt_tos,
elapsed,
)
newrelic.agent.record_custom_metric(
"Custom/email_handler_time", elapsed, newrelic_app
)
newrelic.agent.record_custom_metric(
"Custom/number_incoming_email", 1, newrelic_app
)
return ret
def main(port: int):

49
job_runner.py
View File

@ -12,6 +12,8 @@ from app.config import (
JOB_ONBOARDING_4,
JOB_BATCH_IMPORT,
JOB_DELETE_ACCOUNT,
JOB_DELETE_MAILBOX,
JOB_DELETE_DOMAIN,
)
from app.db import Session
from app.email_utils import (
@ -20,7 +22,7 @@ from app.email_utils import (
)
from app.import_utils import handle_batch_import
from app.log import LOG
from app.models import User, Job, BatchImport
from app.models import User, Job, BatchImport, Mailbox, CustomDomain
from server import create_light_app
@ -163,6 +165,51 @@ if __name__ == "__main__":
render("transactional/account-delete.txt"),
render("transactional/account-delete.html"),
)
elif job.name == JOB_DELETE_MAILBOX:
mailbox_id = job.payload.get("mailbox_id")
mailbox = Mailbox.get(mailbox_id)
if not mailbox:
continue
mailbox_email = mailbox.email
user = mailbox.user
Mailbox.delete(mailbox_id)
Session.commit()
LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email)
send_email(
user.email,
f"Your mailbox {mailbox_email} has been deleted",
f"""Mailbox {mailbox_email} along with its aliases are deleted successfully.
Regards,
SimpleLogin team.
""",
)
elif job.name == JOB_DELETE_DOMAIN:
custom_domain_id = job.payload.get("custom_domain_id")
custom_domain = CustomDomain.get(custom_domain_id)
if not custom_domain:
continue
domain_name = custom_domain.domain
user = custom_domain.user
CustomDomain.delete(custom_domain.id)
Session.commit()
LOG.d("Domain %s deleted", domain_name)
send_email(
user.email,
f"Your domain {domain_name} has been deleted",
f"""Domain {domain_name} along with its aliases are deleted successfully.
Regards,
SimpleLogin team.
""",
)
else:
LOG.e("Unknown job name %s", job.name)

102
local_data/email_tests/bounce.eml Normal file
View File

@ -0,0 +1,102 @@
Received: by mx1.sl.co (Postfix)
id F09806333D; Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
From: mailer-daemon@bounce.sl.co (Mail Delivery System)
Subject: Undelivered Mail Returned to Sender
To: bounce+5352+@sl.co
Auto-Submitted: auto-replied
MIME-Version: 1.0
Content-Type: multipart/report; report-type=delivery-status;
boundary="8A32A6333B.1634202884/mx1.sl.co"
Content-Transfer-Encoding: 8bit
Message-Id: <20211014091444.F09806333D@mx1.sl.co>
This is a MIME-encapsulated message.
--8A32A6333B.1634202884/mx1.sl.co
Content-Description: Notification
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
This is the mail system at host mx1.sl.co.
I'm sorry to have to inform you that your message could not
be delivered to one or more recipients. It's attached below.
For further assistance, please send mail to <postmaster@sl.co>
If you do so, please include this problem report. You can
delete your own text from the attached returned message.
The mail system
<not-existing@gmail.com>: host
gmail-smtp-in.l.google.com[142.251.5.27] said: 550-5.1.1 The email account
that you tried to reach does not exist. Please try 550-5.1.1
double-checking the recipient's email address for typos or 550-5.1.1
unnecessary spaces. Learn more at 550 5.1.1
https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
(in reply to RCPT TO command)
--8A32A6333B.1634202884/mx1.sl.co
Content-Description: Delivery report
Content-Type: message/delivery-status
Reporting-MTA: dns; mx1.sl.co
X-Postfix-Queue-ID: 8A32A6333B
X-Postfix-Sender: rfc822; bounce+5352+@sl.co
Arrival-Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
Final-Recipient: rfc822; not-existing@gmail.com
Original-Recipient: rfc822;not-existing@gmail.com
Action: failed
Status: 5.1.1
Remote-MTA: dns; gmail-smtp-in.l.google.com
Diagnostic-Code: smtp;
550-5.1.1 The email account that you tried to reach does
not exist. Please try 550-5.1.1 double-checking the recipient's email
address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
--8A32A6333B.1634202884/mx1.sl.co
Content-Description: Undelivered Message
Content-Type: message/rfc822
Content-Transfer-Encoding: 8bit
Return-Path: <bounce+5352+@sl.co>
X-SimpleLogin-Client-IP: 90.127.20.84
Received: from 2a01cb00008c9c001a3eeffffec79eea.ipv6.abo.wanadoo.fr
(lfbn-idf1-1-2034-84.w90-127.abo.wanadoo.fr [90.127.20.84])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits))
(No client certificate requested)
by mx1.sl.co (Postfix) with ESMTPS id 8A32A6333B
for <not-existing@gmail.com>;
Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
Content-Type: text/plain;
charset=us-ascii
Content-Transfer-Encoding: 7bit
Mime-Version: 1.0 (Mac OS X Mail 14.0 \(3654.120.0.1.13\))
Subject: bounce 5
Message-Id: <F2EFE3CE-3967-49EC-8639-9A5900230F2E@gmail.com>
X-SimpleLogin-Type: Forward
X-SimpleLogin-EmailLog-ID: 5352
X-SimpleLogin-Envelope-From: sender@gmail.com
X-SimpleLogin-Envelope-To: heyheyalo@sl.co
date: Thu, 14 Oct 2021 09:14:44 -0000
From: "First Last - sender at gmail.com"
<ra+sender.at.gmail.com+bsppvaap@sl.co>
To: heyheyalo@sl.co
List-Unsubscribe: <mailto:unsubsribe@sl.co?subject=26561=>
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=sl.co;
i=@sl.co; q=dns/txt; s=dkim; t=1634202884;
h=message-id : date : subject : from : to;
bh=ktjzaMYZHA8J5baAHC3QyOmFwAAv/MvNtIz1dvmI3V0=;
b=mzf2ZDIVshKSSjw4AQnrOttgRRjzYzZ+49PaPRobt0xFH0E02a2C9Rl/qLEshLHA7amba
8iNTzdTkp9UJquzjk3NwM9GCakmSzd9DmFsalkgeErDAKWNo2O2c7aYDHZlK/sp2vgsIcSO
1w6sp8sVIRr2JrnFPxFOfsOSkSabeOA=
Alo quoi
--8A32A6333B.1634202884/mx1.sl.co--

1
migrations/README Normal file
View File

@ -0,0 +1 @@
Generic single-database configuration.

45
migrations/alembic.ini
View File

@ -1,45 +0,0 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

43
migrations/env.py
View File

@ -1,6 +1,3 @@
from __future__ import with_statement
import logging
from logging.config import fileConfig
from sqlalchemy import engine_from_config
@ -15,23 +12,28 @@ config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from flask import current_app
config.set_main_option(
'sqlalchemy.url', current_app.config.get(
'SQLALCHEMY_DATABASE_URI').replace('%', '%%'))
target_metadata = current_app.extensions['migrate'].db.metadata
import sys
# hack to be able to import Base
# cf https://stackoverflow.com/a/58891735/1428034
sys.path = ['', '..'] + sys.path[1:]
from app.models import Base
from app.config import DB_URI
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
config.set_main_option('sqlalchemy.url', DB_URI)
def run_migrations_offline():
"""Run migrations in 'offline' mode.
@ -47,7 +49,10 @@ def run_migrations_offline():
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
@ -61,29 +66,15 @@ def run_migrations_online():
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():

29
migrations/versions/2021_101415_d750d578b068_.py Normal file
View File

@ -0,0 +1,29 @@
"""empty message
Revision ID: d750d578b068
Revises: 2fbcad5527d7
Create Date: 2021-10-14 15:44:57.816738
"""
import sqlalchemy_utils
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'd750d578b068'
down_revision = '2fbcad5527d7'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('bounce', sa.Column('info', sa.Text(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('bounce', 'info')
# ### end Alembic commands ###

29
migrations/versions/2021_101510_2f1b3c759773_.py Normal file
View File

@ -0,0 +1,29 @@
"""empty message
Revision ID: 2f1b3c759773
Revises: d750d578b068
Create Date: 2021-10-15 10:46:00.389295
"""
import sqlalchemy_utils
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2f1b3c759773'
down_revision = 'd750d578b068'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('metric2', sa.Column('nb_total_bounced_last_24h', sa.Float(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('metric2', 'nb_total_bounced_last_24h')
# ### end Alembic commands ###

11
scripts/new-migration.sh
View File

@ -6,11 +6,14 @@
docker rm -f sl-db
docker run -p 25432:5432 --name sl-db -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=sl -d postgres:13
# run run `flask db upgrade` to upgrade the DB to the latest stage and
env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db upgrade
# sleep a little bit for the db to be ready
sleep 3
# finally `flask db migrate` to generate the migration script.
env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db migrate
# upgrade the DB to the latest stage and
env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic upgrade head
# generate the migration script.
env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic revision --autogenerate
# remove the db
docker rm -f sl-db

21
tests/dashboard/test_custom_domain.py
View File

@ -1,6 +1,7 @@
from flask import url_for
from app.db import Session
from app.models import Mailbox
from tests.utils import login
@ -36,3 +37,23 @@ def test_add_domain_same_as_user_email(flask_client):
b"You cannot add a domain that you are currently using for your personal email"
in r.data
)
def test_add_domain_used_in_mailbox(flask_client):
"""cannot add domain if it has been used in a verified mailbox"""
user = login(flask_client)
user.lifetime = True
Session.commit()
Mailbox.create(
user_id=user.id, email="mailbox@new-domain.com", verified=True, commit=True
)
r = flask_client.post(
url_for("dashboard.custom_domain"),
data={"form-name": "create", "domain": "new-domain.com"},
follow_redirects=True,
)
assert r.status_code == 200
assert b"new-domain.com already used in a SimpleLogin mailbox" in r.data

23
tests/test_email_utils.py
View File

@ -1,9 +1,10 @@
import email
import os
from email.message import EmailMessage
import arrow
from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL
from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR
from app.db import Session
from app.email_utils import (
get_email_domain_part,
@ -31,6 +32,8 @@ from app.email_utils import (
should_ignore_bounce,
get_header_unicode,
parse_full_address,
get_orig_message_from_bounce,
get_mailbox_bounce_info,
)
from app.models import User, CustomDomain, Alias, Contact, EmailLog, IgnoreBounceSender
@ -748,3 +751,21 @@ def test_should_ignore_bounce(flask_client):
def test_get_header_unicode():
assert get_header_unicode("ab@cd.com") == "ab@cd.com"
assert get_header_unicode("=?utf-8?B?w6nDqQ==?=@example.com") == "éé@example.com"
def test_get_orig_message_from_bounce():
with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f:
bounce_report = email.message_from_file(f)
orig_msg = get_orig_message_from_bounce(bounce_report)
assert orig_msg["X-SimpleLogin-Type"] == "Forward"
assert orig_msg["X-SimpleLogin-Envelope-From"] == "sender@gmail.com"
def test_get_mailbox_bounce_info():
with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f:
bounce_report = email.message_from_file(f)
orig_msg = get_mailbox_bounce_info(bounce_report)
assert orig_msg["Final-Recipient"] == "rfc822; not-existing@gmail.com"
assert orig_msg["Original-Recipient"] == "rfc822;not-existing@gmail.com"