From 37ca4eaf20f2da7cdceb8255e85d8384f1c119d8 Mon Sep 17 00:00:00 2001 From: doanguyen Date: Tue, 31 Dec 2019 11:11:06 +0100 Subject: [PATCH 01/76] working on paginate alias log page --- .../templates/dashboard/alias_log.html | 10 ++++++-- app/dashboard/views/alias_log.py | 23 ++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/app/dashboard/templates/dashboard/alias_log.html b/app/dashboard/templates/dashboard/alias_log.html index d6b3c71b..59c40d06 100644 --- a/app/dashboard/templates/dashboard/alias_log.html +++ b/app/dashboard/templates/dashboard/alias_log.html @@ -7,7 +7,7 @@ {% endblock %} {% block default_content %} - diff --git a/app/dashboard/views/lifetime_licence.py b/app/dashboard/views/lifetime_licence.py new file mode 100644 index 00000000..66511abb --- /dev/null +++ b/app/dashboard/views/lifetime_licence.py @@ -0,0 +1,57 @@ +from flask import render_template, flash, redirect, url_for +from flask_login import login_required, current_user +from flask_wtf import FlaskForm +from wtforms import StringField, validators + +from app.config import ( + PADDLE_VENDOR_ID, + PADDLE_MONTHLY_PRODUCT_ID, + PADDLE_YEARLY_PRODUCT_ID, + URL, + ADMIN_EMAIL, +) +from app.dashboard.base import dashboard_bp +from app.email_utils import send_email +from app.extensions import db +from app.models import LifetimeCoupon + + +class CouponForm(FlaskForm): + code = StringField("Coupon Code", validators=[validators.DataRequired()]) + + +@dashboard_bp.route("/lifetime_licence", methods=["GET", "POST"]) +@login_required +def lifetime_licence(): + # sanity check: make sure this page is only for free user + if current_user.is_premium(): + flash("You are already a premium user", "warning") + return redirect(url_for("dashboard.index")) + + coupon_form = CouponForm() + + if coupon_form.validate_on_submit(): + code = coupon_form.code.data + + coupon = LifetimeCoupon.get_by(code=code) + + if coupon and coupon.nb_used > 0: + coupon.nb_used -= 1 + current_user.lifetime = True + db.session.commit() + + # notify admin + send_email( + ADMIN_EMAIL, + subject=f"User {current_user.id} used lifetime coupon. Coupon nb_used: {coupon.nb_used}", + plaintext="", + html="", + ) + + flash("You are upgraded to lifetime premium!", "success") + return redirect(url_for("dashboard.index")) + + else: + flash(f"Code *{code}* expired or invalid", "warning") + + return render_template("dashboard/lifetime_licence.html", coupon_form=coupon_form) diff --git a/app/dashboard/views/pricing.py b/app/dashboard/views/pricing.py index aa74ea34..d29891bc 100644 --- a/app/dashboard/views/pricing.py +++ b/app/dashboard/views/pricing.py @@ -1,23 +1,13 @@ from flask import render_template, flash, redirect, url_for from flask_login import login_required, current_user -from flask_wtf import FlaskForm -from wtforms import StringField, validators from app.config import ( PADDLE_VENDOR_ID, PADDLE_MONTHLY_PRODUCT_ID, PADDLE_YEARLY_PRODUCT_ID, URL, - ADMIN_EMAIL, ) from app.dashboard.base import dashboard_bp -from app.email_utils import send_email -from app.extensions import db -from app.models import LifetimeCoupon - - -class CouponForm(FlaskForm): - code = StringField("Coupon Code", validators=[validators.DataRequired()]) @dashboard_bp.route("/pricing", methods=["GET", "POST"]) @@ -28,39 +18,12 @@ def pricing(): flash("You are already a premium user", "warning") return redirect(url_for("dashboard.index")) - coupon_form = CouponForm() - - if coupon_form.validate_on_submit(): - code = coupon_form.code.data - - coupon = LifetimeCoupon.get_by(code=code) - - if coupon and coupon.nb_used > 0: - coupon.nb_used -= 1 - current_user.lifetime = True - db.session.commit() - - # notify admin - send_email( - ADMIN_EMAIL, - subject=f"User {current_user.id} used lifetime coupon. Coupon nb_used: {coupon.nb_used}", - plaintext="", - html="", - ) - - flash("You are upgraded to lifetime premium!", "success") - return redirect(url_for("dashboard.index")) - - else: - flash(f"Coupon *{code}* expired or invalid", "warning") - return render_template( "dashboard/pricing.html", PADDLE_VENDOR_ID=PADDLE_VENDOR_ID, PADDLE_MONTHLY_PRODUCT_ID=PADDLE_MONTHLY_PRODUCT_ID, PADDLE_YEARLY_PRODUCT_ID=PADDLE_YEARLY_PRODUCT_ID, success_url=URL + "/dashboard/subscription_success", - coupon_form=coupon_form, ) From 35aa8f14381fa2ee5ecf0cece2451966b9d088b2 Mon Sep 17 00:00:00 2001 From: Son NK Date: Fri, 3 Jan 2020 23:13:52 +0100 Subject: [PATCH 44/76] add GoatCounter analytics --- server.py | 18 ++++++++++++++++++ templates/base.html | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/server.py b/server.py index a9f29264..08d95c42 100644 --- a/server.py +++ b/server.py @@ -89,6 +89,7 @@ def create_app() -> Flask: init_admin(app) setup_paddle_callback(app) + setup_do_not_track(app) if FLASK_PROFILER_PATH: LOG.d("Enable flask-profiler") @@ -401,6 +402,23 @@ def init_admin(app): admin.add_view(SLModelView(ClientUser, db.session)) +def setup_do_not_track(app): + @app.route("/dnt") + def do_not_track(): + return """ + + """ + + if __name__ == "__main__": app = create_app() diff --git a/templates/base.html b/templates/base.html index 48614e93..8c49aeda 100644 --- a/templates/base.html +++ b/templates/base.html @@ -128,5 +128,30 @@ {% block script %} {% endblock %} + + \ No newline at end of file From 4208ba379f81c6569bb288ee160c75543dd7e786 Mon Sep 17 00:00:00 2001 From: Son NK Date: Fri, 3 Jan 2020 23:42:35 +0100 Subject: [PATCH 45/76] Fix user could go to MFA page directly --- app/auth/views/mfa.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/auth/views/mfa.py b/app/auth/views/mfa.py index cc21727d..6565664c 100644 --- a/app/auth/views/mfa.py +++ b/app/auth/views/mfa.py @@ -17,11 +17,18 @@ class OtpTokenForm(FlaskForm): @auth_bp.route("/mfa", methods=["GET", "POST"]) def mfa(): # passed from login page - user_id = session[MFA_USER_ID] + user_id = session.get(MFA_USER_ID) + + # user access this page directly without passing by login page + if not user_id: + flash("Unknown error, redirect back to main page", "warning") + return redirect(url_for("dashboard.index")) + user = User.get(user_id) - if not user.enable_otp: - raise Exception("Only user with MFA enabled should go to this page. %s", user) + if not (user and user.enable_otp): + flash("Only user with MFA enabled should go to this page", "warning") + return redirect(url_for("dashboard.index")) otp_token_form = OtpTokenForm() next_url = request.args.get("next") From 837ab8258e82249337e7ffe5b63e4bc7f4e8372a Mon Sep 17 00:00:00 2001 From: Son NK Date: Fri, 3 Jan 2020 23:50:34 +0100 Subject: [PATCH 46/76] redirect to login page instead --- app/auth/views/mfa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/auth/views/mfa.py b/app/auth/views/mfa.py index 6565664c..3920c33b 100644 --- a/app/auth/views/mfa.py +++ b/app/auth/views/mfa.py @@ -22,13 +22,13 @@ def mfa(): # user access this page directly without passing by login page if not user_id: flash("Unknown error, redirect back to main page", "warning") - return redirect(url_for("dashboard.index")) + return redirect(url_for("auth.login")) user = User.get(user_id) if not (user and user.enable_otp): flash("Only user with MFA enabled should go to this page", "warning") - return redirect(url_for("dashboard.index")) + return redirect(url_for("auth.login")) otp_token_form = OtpTokenForm() next_url = request.args.get("next") From d6aa6e7b9490ed5b87108498292c01de58b9d3ab Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 4 Jan 2020 10:24:01 +0100 Subject: [PATCH 47/76] Make sure to user lowercase for user email --- app/auth/views/facebook.py | 4 +++- app/auth/views/github.py | 2 +- app/auth/views/google.py | 4 +++- app/auth/views/register.py | 4 +++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/auth/views/facebook.py b/app/auth/views/facebook.py index 6ada5445..c552cba6 100644 --- a/app/auth/views/facebook.py +++ b/app/auth/views/facebook.py @@ -103,7 +103,9 @@ def facebook_callback(): # create user else: LOG.d("create facebook user with %s", facebook_user_data) - user = User.create(email=email, name=facebook_user_data["name"], activated=True) + user = User.create( + email=email.lower(), name=facebook_user_data["name"], activated=True + ) if picture_url: LOG.d("set user profile picture to %s", picture_url) diff --git a/app/auth/views/github.py b/app/auth/views/github.py index c81a2b29..51bf0763 100644 --- a/app/auth/views/github.py +++ b/app/auth/views/github.py @@ -86,7 +86,7 @@ def github_callback(): if not user: LOG.d("create github user") user = User.create( - email=email, name=github_user_data.get("name") or "", activated=True + email=email.lower(), name=github_user_data.get("name") or "", activated=True ) db.session.commit() login_user(user) diff --git a/app/auth/views/google.py b/app/auth/views/google.py index 60480b1d..69dc9196 100644 --- a/app/auth/views/google.py +++ b/app/auth/views/google.py @@ -93,7 +93,9 @@ def google_callback(): # create user else: LOG.d("create google user with %s", google_user_data) - user = User.create(email=email, name=google_user_data["name"], activated=True) + user = User.create( + email=email.lower(), name=google_user_data["name"], activated=True + ) if picture_url: LOG.d("set user profile picture to %s", picture_url) diff --git a/app/auth/views/register.py b/app/auth/views/register.py index 7166c301..bb5b3b11 100644 --- a/app/auth/views/register.py +++ b/app/auth/views/register.py @@ -46,7 +46,9 @@ def register(): else: LOG.debug("create user %s", form.email.data) user = User.create( - email=form.email.data, name=form.name.data, password=form.password.data + email=form.email.data.lower(), + name=form.name.data, + password=form.password.data, ) db.session.commit() From 0dc901d05d6a10f8500794e41106bdd643b72c3f Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 4 Jan 2020 10:25:19 +0100 Subject: [PATCH 48/76] make email lowercase before processing them --- email_handler.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/email_handler.py b/email_handler.py index 00608592..bd70b37d 100644 --- a/email_handler.py +++ b/email_handler.py @@ -31,7 +31,7 @@ It should contain the following info: """ import time -from email.message import EmailMessage +from email.message import Message from email.parser import Parser from email.policy import SMTPUTF8 from smtplib import SMTP @@ -85,11 +85,11 @@ class MailHandler: smtp = SMTP(POSTFIX_SERVER, 25) msg = Parser(policy=SMTPUTF8).parsestr(message_data) + rcpt_to = envelope.rcpt_tos[0].lower() + # Reply case # reply+ or ra+ (reverse-alias) prefix - if envelope.rcpt_tos[0].startswith("reply+") or envelope.rcpt_tos[0].startswith( - "ra+" - ): + if rcpt_to.startswith("reply+") or rcpt_to.startswith("ra+"): LOG.debug("Reply phase") app = new_app() @@ -102,9 +102,9 @@ class MailHandler: with app.app_context(): return self.handle_forward(envelope, smtp, msg) - def handle_forward(self, envelope, smtp: SMTP, msg: EmailMessage) -> str: + def handle_forward(self, envelope, smtp: SMTP, msg: Message) -> str: """return *status_code message*""" - alias = envelope.rcpt_tos[0] # alias@SL + alias = envelope.rcpt_tos[0].lower() # alias@SL gen_email = GenEmail.get_by(email=alias) if not gen_email: @@ -212,8 +212,8 @@ class MailHandler: db.session.commit() return "250 Message accepted for delivery" - def handle_reply(self, envelope, smtp: SMTP, msg: EmailMessage) -> str: - reply_email = envelope.rcpt_tos[0] + def handle_reply(self, envelope, smtp: SMTP, msg: Message) -> str: + reply_email = envelope.rcpt_tos[0].lower() # reply_email must end with EMAIL_DOMAIN if not reply_email.endswith(EMAIL_DOMAIN): @@ -230,7 +230,7 @@ class MailHandler: return "550 alias unknown by SimpleLogin" user_email = forward_email.gen_email.user.email - if envelope.mail_from != user_email: + if envelope.mail_from.lower() != user_email.lower(): LOG.error( f"Reply email can only be used by user email. Actual mail_from: %s. User email %s", envelope.mail_from, @@ -293,7 +293,7 @@ class MailHandler: return "250 Message accepted for delivery" -def add_or_replace_header(msg: EmailMessage, header: str, value: str): +def add_or_replace_header(msg: Message, header: str, value: str): try: msg.add_header(header, value) except ValueError: From 40c2040ddc78dac3310708e3b969f4b5198005a4 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 4 Jan 2020 10:58:19 +0100 Subject: [PATCH 49/76] use google nameserver --- app/dns_utils.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/app/dns_utils.py b/app/dns_utils.py index cb0f1799..a0e08b95 100644 --- a/app/dns_utils.py +++ b/app/dns_utils.py @@ -6,7 +6,12 @@ def get_mx_domains(hostname) -> [(int, str)]: domain name ends with a "." at the end. """ try: - answers = dns.resolver.query(hostname, "MX") + my_resolver = dns.resolver.Resolver() + + # 8.8.8.8 is Google's public DNS server + my_resolver.nameservers = ['8.8.8.8'] + + answers = my_resolver.query(hostname, "MX") except Exception: return [] @@ -27,7 +32,12 @@ _include_spf = "include:" def get_spf_domain(hostname) -> [str]: """return all domains listed in *include:*""" try: - answers = dns.resolver.query(hostname, "TXT") + my_resolver = dns.resolver.Resolver() + + # 8.8.8.8 is Google's public DNS server + my_resolver.nameservers = ['8.8.8.8'] + + answers = my_resolver.query(hostname, "TXT") except Exception: return [] @@ -48,7 +58,12 @@ def get_spf_domain(hostname) -> [str]: def get_txt_record(hostname) -> [str]: try: - answers = dns.resolver.query(hostname, "TXT") + my_resolver = dns.resolver.Resolver() + + # 8.8.8.8 is Google's public DNS server + my_resolver.nameservers = ['8.8.8.8'] + + answers = my_resolver.query(hostname, "TXT") except Exception: return [] @@ -66,7 +81,12 @@ def get_txt_record(hostname) -> [str]: def get_dkim_record(hostname) -> str: """query the dkim._domainkey.{hostname} record and returns its value""" try: - answers = dns.resolver.query(f"dkim._domainkey.{hostname}", "TXT") + my_resolver = dns.resolver.Resolver() + + # 8.8.8.8 is Google's public DNS server + my_resolver.nameservers = ['8.8.8.8'] + + answers = my_resolver.query(f"dkim._domainkey.{hostname}", "TXT") except Exception: return "" From 47f691cacf97cef0f6d16fa91784f2c8c39ae622 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 4 Jan 2020 11:00:59 +0100 Subject: [PATCH 50/76] fix formatting --- app/dns_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/dns_utils.py b/app/dns_utils.py index a0e08b95..493a3feb 100644 --- a/app/dns_utils.py +++ b/app/dns_utils.py @@ -9,7 +9,7 @@ def get_mx_domains(hostname) -> [(int, str)]: my_resolver = dns.resolver.Resolver() # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ['8.8.8.8'] + my_resolver.nameservers = ["8.8.8.8"] answers = my_resolver.query(hostname, "MX") except Exception: @@ -35,7 +35,7 @@ def get_spf_domain(hostname) -> [str]: my_resolver = dns.resolver.Resolver() # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ['8.8.8.8'] + my_resolver.nameservers = ["8.8.8.8"] answers = my_resolver.query(hostname, "TXT") except Exception: @@ -61,7 +61,7 @@ def get_txt_record(hostname) -> [str]: my_resolver = dns.resolver.Resolver() # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ['8.8.8.8'] + my_resolver.nameservers = ["8.8.8.8"] answers = my_resolver.query(hostname, "TXT") except Exception: @@ -84,7 +84,7 @@ def get_dkim_record(hostname) -> str: my_resolver = dns.resolver.Resolver() # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ['8.8.8.8'] + my_resolver.nameservers = ["8.8.8.8"] answers = my_resolver.query(f"dkim._domainkey.{hostname}", "TXT") except Exception: From 41329782a23ed2c871881c2ab2492054a611393a Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 19:01:38 +0100 Subject: [PATCH 51/76] refactor dns_utils and add test_dns_utils --- app/dns_utils.py | 42 ++++++++++++++--------------------------- tests/test_dns_utils.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 tests/test_dns_utils.py diff --git a/app/dns_utils.py b/app/dns_utils.py index 493a3feb..aed45c0f 100644 --- a/app/dns_utils.py +++ b/app/dns_utils.py @@ -1,17 +1,21 @@ import dns.resolver +def _get_dns_resolver(): + my_resolver = dns.resolver.Resolver() + + # 8.8.8.8 is Google's public DNS server + my_resolver.nameservers = ["8.8.8.8"] + + return my_resolver + + def get_mx_domains(hostname) -> [(int, str)]: """return list of (priority, domain name). domain name ends with a "." at the end. """ try: - my_resolver = dns.resolver.Resolver() - - # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ["8.8.8.8"] - - answers = my_resolver.query(hostname, "MX") + answers = _get_dns_resolver().query(hostname, "MX") except Exception: return [] @@ -32,12 +36,7 @@ _include_spf = "include:" def get_spf_domain(hostname) -> [str]: """return all domains listed in *include:*""" try: - my_resolver = dns.resolver.Resolver() - - # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ["8.8.8.8"] - - answers = my_resolver.query(hostname, "TXT") + answers = _get_dns_resolver().query(hostname, "TXT") except Exception: return [] @@ -58,22 +57,14 @@ def get_spf_domain(hostname) -> [str]: def get_txt_record(hostname) -> [str]: try: - my_resolver = dns.resolver.Resolver() - - # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ["8.8.8.8"] - - answers = my_resolver.query(hostname, "TXT") + answers = _get_dns_resolver().query(hostname, "TXT") except Exception: return [] ret = [] for a in answers: # type: dns.rdtypes.ANY.TXT.TXT - for record in a.strings: - record = record.decode() # record is bytes - - ret.append(a) + ret.append(a) return ret @@ -81,12 +72,7 @@ def get_txt_record(hostname) -> [str]: def get_dkim_record(hostname) -> str: """query the dkim._domainkey.{hostname} record and returns its value""" try: - my_resolver = dns.resolver.Resolver() - - # 8.8.8.8 is Google's public DNS server - my_resolver.nameservers = ["8.8.8.8"] - - answers = my_resolver.query(f"dkim._domainkey.{hostname}", "TXT") + answers = _get_dns_resolver().query(f"dkim._domainkey.{hostname}", "TXT") except Exception: return "" diff --git a/tests/test_dns_utils.py b/tests/test_dns_utils.py new file mode 100644 index 00000000..e03acbb6 --- /dev/null +++ b/tests/test_dns_utils.py @@ -0,0 +1,30 @@ +from app.dns_utils import * + +# use our own domain for test +_DOMAIN = "simplelogin.io" + + +def test_get_mx_domains(): + r = get_mx_domains(_DOMAIN) + + assert len(r) > 0 + + for x in r: + assert x[0] > 0 + assert x[1] + + +def test_get_spf_domain(): + r = get_spf_domain(_DOMAIN) + assert r == ["simplelogin.co"] + + +def test_get_txt_record(): + + r = get_txt_record(_DOMAIN) + assert len(r) > 0 + + +def test_get_dkim_record(): + r = get_dkim_record(_DOMAIN) + assert r.startswith("v=DKIM1; k=rsa;") From 4e84815375b79e4913863d589dd528f1b70e6da0 Mon Sep 17 00:00:00 2001 From: doanguyen Date: Sun, 5 Jan 2020 19:45:29 +0100 Subject: [PATCH 52/76] let debug configurable --- app/config.py | 2 ++ server.py | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/config.py b/app/config.py index ae2082ee..11608748 100644 --- a/app/config.py +++ b/app/config.py @@ -30,6 +30,8 @@ COLOR_LOG = "COLOR_LOG" in os.environ # Allow user to have 1 year of premium: set the expiration_date to 1 year more PROMO_CODE = "SIMPLEISBETTER" +# Debug mode +DEBUG = os.environ['DEBUG'] # Server url URL = os.environ["URL"] print(">>> URL:", URL) diff --git a/server.py b/server.py index 08d95c42..5425b926 100644 --- a/server.py +++ b/server.py @@ -18,6 +18,7 @@ from app.admin_model import SLModelView, SLAdminIndexView from app.api.base import api_bp from app.auth.base import auth_bp from app.config import ( + DEBUG, DB_URI, FLASK_SECRET, SENTRY_DSN, @@ -67,6 +68,8 @@ os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" def create_app() -> Flask: app = Flask(__name__) app.url_map.strict_slashes = False + app.debug = DEBUG + os.environ["FLASK_DEBUG"] = DEBUG app.config["SQLALCHEMY_DATABASE_URI"] = DB_URI app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False @@ -422,8 +425,6 @@ window.location.href = "/"; if __name__ == "__main__": app = create_app() - app.debug = True - # enable flask toolbar # app.config["DEBUG_TB_PROFILER_ENABLED"] = True # app.config["DEBUG_TB_INTERCEPT_REDIRECTS"] = False @@ -444,6 +445,6 @@ if __name__ == "__main__": context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) context.load_cert_chain("local_data/cert.pem", "local_data/key.pem") - app.run(debug=True, host="0.0.0.0", port=7777, ssl_context=context) + app.run(host="0.0.0.0", port=7777, ssl_context=context) else: - app.run(debug=True, host="0.0.0.0", port=7777) + app.run(host="0.0.0.0", port=7777) From 96da841062d77c13c6edfc31f95479d391c66527 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 20:47:09 +0100 Subject: [PATCH 53/76] add /api/v2/alias/options that flattens the response --- app/api/views/alias_options.py | 75 +++++++++++++++++++++++++++++++++ tests/api/test_alias_options.py | 45 ++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/app/api/views/alias_options.py b/app/api/views/alias_options.py index 07b21c09..d3d3dcf4 100644 --- a/app/api/views/alias_options.py +++ b/app/api/views/alias_options.py @@ -78,3 +78,78 @@ def options(): ret["custom"]["suffixes"] = list(reversed(ret["custom"]["suffixes"])) return jsonify(ret) + + +@api_bp.route("/v2/alias/options") +@cross_origin() +@verify_api_key +def options_v2(): + """ + Return what options user has when creating new alias. + Input: + a valid api-key in "Authentication" header and + optional "hostname" in args + Output: cf README + can_create: bool + suffixes: [str] + prefix_suggestion: str + existing: [str] + recommendation: Optional dict + alias: str + hostname: str + + + """ + user = g.user + hostname = request.args.get("hostname") + + ret = { + "existing": [ + ge.email for ge in GenEmail.query.filter_by(user_id=user.id, enabled=True) + ], + "can_create": user.can_create_new_alias(), + "suffixes": [], + "prefix_suggestion": "", + } + + # recommendation alias if exist + if hostname: + # put the latest used alias first + q = ( + db.session.query(AliasUsedOn, GenEmail, User) + .filter( + AliasUsedOn.gen_email_id == GenEmail.id, + GenEmail.user_id == user.id, + AliasUsedOn.hostname == hostname, + ) + .order_by(desc(AliasUsedOn.created_at)) + ) + + r = q.first() + if r: + _, alias, _ = r + LOG.d("found alias %s %s %s", alias, hostname, user) + ret["recommendation"] = {"alias": alias.email, "hostname": hostname} + + # custom alias suggestion and suffix + if hostname: + # keep only the domain name of hostname, ignore TLD and subdomain + # for ex www.groupon.com -> groupon + domain_name = hostname + if "." in hostname: + parts = hostname.split(".") + domain_name = parts[-2] + domain_name = convert_to_id(domain_name) + ret["prefix_suggestion"] = domain_name + + # maybe better to make sure the suffix is never used before + # but this is ok as there's a check when creating a new custom alias + ret["suffixes"] = [f".{random_word()}@{EMAIL_DOMAIN}"] + + for custom_domain in user.verified_custom_domains(): + ret["suffixes"].append("@" + custom_domain.domain) + + # custom domain should be put first + ret["suffixes"] = list(reversed(ret["suffixes"])) + + return jsonify(ret) diff --git a/tests/api/test_alias_options.py b/tests/api/test_alias_options.py index 6f14901e..b7ac4e72 100644 --- a/tests/api/test_alias_options.py +++ b/tests/api/test_alias_options.py @@ -50,3 +50,48 @@ def test_different_scenarios(flask_client): ) assert r.json["recommendation"]["alias"] == alias.email assert r.json["recommendation"]["hostname"] == "www.test.com" + + +def test_different_scenarios_v2(flask_client): + user = User.create( + email="a@b.c", password="password", name="Test User", activated=True + ) + db.session.commit() + + # create api_key + api_key = ApiKey.create(user.id, "for test") + db.session.commit() + + # <<< without hostname >>> + r = flask_client.get( + url_for("api.options_v2"), headers={"Authentication": api_key.code} + ) + + assert r.status_code == 200 + # {'can_create': True, 'existing': ['my-first-alias.chat@sl.local'], 'prefix_suggestion': '', 'suffixes': ['.meo@sl.local']} + + assert r.json["can_create"] + assert len(r.json["existing"]) == 1 + assert r.json["suffixes"] + assert r.json["prefix_suggestion"] == "" # no hostname => no suggestion + + # <<< with hostname >>> + r = flask_client.get( + url_for("api.options_v2", hostname="www.test.com"), + headers={"Authentication": api_key.code}, + ) + + assert r.json["prefix_suggestion"] == "test" + + # <<< with recommendation >>> + alias = GenEmail.create_new(user.id, prefix="test") + db.session.commit() + AliasUsedOn.create(gen_email_id=alias.id, hostname="www.test.com") + db.session.commit() + + r = flask_client.get( + url_for("api.options_v2", hostname="www.test.com"), + headers={"Authentication": api_key.code}, + ) + assert r.json["recommendation"]["alias"] == alias.email + assert r.json["recommendation"]["hostname"] == "www.test.com" From 5b60869a3933bef5c426192872b0bf3772bdeb3c Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 20:47:56 +0100 Subject: [PATCH 54/76] Update API doc --- README.md | 90 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 21c7f764..578554b2 100644 --- a/README.md +++ b/README.md @@ -427,52 +427,74 @@ In every request, the extension sends - the `API Code` is set in `Authentication` header. The check is done via the `verify_api_key` wrapper, implemented in `app/api/base.py` -- the current website `hostname` which is the website subdomain name + domain name. For ex, if user is on `http://dashboard.example.com/path1/path2?query`, the subdomain is `dashboard.example.com`. This information is important to know where an alias is used in order to proposer to user the same alias if they want to create on alias on the same website in the future. The `hostname` is passed in the request query `?hostname=`, see `app/api/views/alias_options.py` for an example. +- (Optional but recommended) `hostname` passed in query string. hostname is the the URL hostname (cf https://en.wikipedia.org/wiki/URL), for ex if URL is http://www.example.com/index.html then the hostname is `www.example.com`. This information is important to know where an alias is used in order to suggest user the same alias if they want to create on alias on the same website in the future. -Currently, the latest extension uses the two following endpoints : +If error, the API returns 4** with body containing the error message, for example: -- `/alias/options`: returns what to suggest to user when they open the extension. - -``` -GET /alias/options hostname?="www.groupon.com" - -Response: a json with following structure. ? means optional field. - recommendation?: - alias: www_groupon_com@simplelogin.co - hostname: www.groupon.com - - custom: - suggestion: groupon - suffix: [@my_domain.com, .abcde@simplelogin.co] - - can_create_custom: true - - existing: - [email1, email2, ...] +```json +{"error": "request body cannot be empty"} ``` -- `/alias/custom/new`: allows user to create a new custom alias. +The error message is used mostly for debugging and should never be displayed to user as-is. -To try out the endpoint, you can use the following command. The command uses [httpie](https://httpie.org). -Make sure to replace `{api_key}` by your API Key obtained on https://app.simplelogin.io/dashboard/api_key +#### GET /api/v2/alias/options -``` -http https://app.simplelogin.io/api/alias/options \ - Authentication:{api_key} \ - hostname==www.google.com +User alias info and suggestion. Used by the first extension screen when user opens the extension. + +Input: +- `Authentication` header that contains the api key +- (Optional but recommended) `hostname` passed in query string. + +Output: a json with the following field: +- can_create: boolean. Whether user can create new alias +- suffixes: list of string. List of alias `suffix` that user can use. If user doesn't have custom domain, this list has a single element which is the alias default domain (simplelogin.co). +- prefix_suggestion: string. Suggestion for the `alias prefix`. Usually this is the website name extracted from `hostname`. If no `hostname`, then the `prefix_suggestion` is empty. +- existing: list of string. List of existing alias. +- recommendation: optional field, dictionary. If an alias is already used for this website, the recommendation will be returned. There are 2 subfields in `recommendation`: `alias` which is the recommended alias and `hostname` is the website on which this alias is used before. + +For ex: +```json +{ + "can_create": true, + "existing": [ + "my-first-alias.meo@sl.local", + "e1.cat@sl.local", + "e2.chat@sl.local", + "e3.cat@sl.local" + ], + "prefix_suggestion": "test", + "recommendation": { + "alias": "e1.cat@sl.local", + "hostname": "www.test.com" + }, + "suffixes": [ + "@very-long-domain.com.net.org", + "@ab.cd", + ".cat@sl.local" + ] +} ``` -``` -POST /alias/custom/new - prefix: www_groupon_com - suffix: @my_domain.com +#### POST /alias/custom/new -Response: - 201 -> OK {alias: "www_groupon_com@my_domain.com"} - 409 -> duplicated +Create a new custom alias. +Input: +- `Authentication` header that contains the api key +- (Optional but recommended) `hostname` passed in query string +- Request Message Body in json (`Content-Type` is `application/json`) + - alias_prefix: string. The first part of the alias that user can choose. + - alias_suffix: should be one of the suffixes returned in the `GET /api/v2/alias/options` endpoint. + +Output: +If success, 201 with the new alias, for example + +```json +{alias: "www_groupon_com@my_domain.com"} ``` +409 if the alias is already created. + ### Database migration The database migration is handled by `alembic` From d322d543af2a3187f44aef42676b6bfb79eb6577 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 20:48:32 +0100 Subject: [PATCH 55/76] add more check to new custom alias --- app/api/views/new_custom_alias.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/api/views/new_custom_alias.py b/app/api/views/new_custom_alias.py index d3dd2b14..86b2919a 100644 --- a/app/api/views/new_custom_alias.py +++ b/app/api/views/new_custom_alias.py @@ -40,8 +40,11 @@ def new_custom_alias(): hostname = request.args.get("hostname") data = request.get_json() - alias_prefix = data["alias_prefix"] - alias_suffix = data["alias_suffix"] + if not data: + return jsonify(error="request body cannot be empty"), 400 + + alias_prefix = data.get("alias_prefix", "") + alias_suffix = data.get("alias_suffix", "") # make sure alias_prefix is not empty alias_prefix = alias_prefix.strip() From deba806d0fce37c08b7b00a8293b0d262459759a Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 21:09:54 +0100 Subject: [PATCH 56/76] Fix readme: "error" could be shown to user as-is --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 578554b2..2556fd46 100644 --- a/README.md +++ b/README.md @@ -432,10 +432,15 @@ In every request, the extension sends If error, the API returns 4** with body containing the error message, for example: ```json -{"error": "request body cannot be empty"} +{ + "error": "request body cannot be empty" +} ``` -The error message is used mostly for debugging and should never be displayed to user as-is. +The error message could be displayed to user as-is, for example for when user exceeds their alias quota. +Some errors should be fixed during development however: for example error like `request body cannot be empty` is there to catch development error and should never be shown to user. + +All following endpoint return `401` status code if the API Key is incorrect. #### GET /api/v2/alias/options @@ -490,7 +495,9 @@ Output: If success, 201 with the new alias, for example ```json -{alias: "www_groupon_com@my_domain.com"} +{ + "alias": "www_groupon_com@my_domain.com" +} ``` 409 if the alias is already created. From c6db8db4a1fffc1fb75fc07700dff068dcf90607 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 21:14:40 +0100 Subject: [PATCH 57/76] Improve error message --- app/api/views/new_custom_alias.py | 4 ++-- tests/api/test_new_custom_alias.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/views/new_custom_alias.py b/app/api/views/new_custom_alias.py index 86b2919a..767020f5 100644 --- a/app/api/views/new_custom_alias.py +++ b/app/api/views/new_custom_alias.py @@ -3,7 +3,7 @@ from flask import jsonify, request from flask_cors import cross_origin from app.api.base import api_bp, verify_api_key -from app.config import EMAIL_DOMAIN +from app.config import EMAIL_DOMAIN, MAX_NB_EMAIL_FREE_PLAN from app.extensions import db from app.log import LOG from app.models import GenEmail, AliasUsedOn @@ -31,7 +31,7 @@ def new_custom_alias(): return ( jsonify( error="You have reached the limitation of a free account with the maximum of " - "3 custom aliases, please upgrade your plan to create more custom aliases" + f"{MAX_NB_EMAIL_FREE_PLAN} aliases, please upgrade your plan to create more aliases" ), 400, ) diff --git a/tests/api/test_new_custom_alias.py b/tests/api/test_new_custom_alias.py index 35ae512c..4dabd645 100644 --- a/tests/api/test_new_custom_alias.py +++ b/tests/api/test_new_custom_alias.py @@ -48,5 +48,5 @@ def test_out_of_quota(flask_client): assert r.status_code == 400 assert r.json == { - "error": "You have reached the limitation of a free account with the maximum of 3 custom aliases, please upgrade your plan to create more custom aliases" + "error": "You have reached the limitation of a free account with the maximum of 3 aliases, please upgrade your plan to create more aliases" } From 377e6c657d5809a2dea0481cee7687e3e3e4d73a Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 21:15:16 +0100 Subject: [PATCH 58/76] add /api/alias/random/new --- README.md | 19 +++++++++-- app/api/__init__.py | 2 +- app/api/views/new_random_alias.py | 43 ++++++++++++++++++++++++ tests/api/test_new_random_alias.py | 52 ++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 app/api/views/new_random_alias.py create mode 100644 tests/api/test_new_random_alias.py diff --git a/README.md b/README.md index 2556fd46..a2c37df2 100644 --- a/README.md +++ b/README.md @@ -480,7 +480,7 @@ For ex: } ``` -#### POST /alias/custom/new +#### POST /api/alias/custom/new Create a new custom alias. @@ -500,7 +500,22 @@ If success, 201 with the new alias, for example } ``` -409 if the alias is already created. +#### POST /api/alias/random/new + +Create a new random alias. + +Input: +- `Authentication` header that contains the api key +- (Optional but recommended) `hostname` passed in query string + +Output: +If success, 201 with the new alias, for example + +```json +{ + "alias": "www_groupon_com@my_domain.com" +} +``` ### Database migration diff --git a/app/api/__init__.py b/app/api/__init__.py index 7ac2561f..612893b0 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -1 +1 @@ -from .views import alias_options, new_custom_alias +from .views import alias_options, new_custom_alias, new_random_alias diff --git a/app/api/views/new_random_alias.py b/app/api/views/new_random_alias.py new file mode 100644 index 00000000..7afe1f6d --- /dev/null +++ b/app/api/views/new_random_alias.py @@ -0,0 +1,43 @@ +from flask import g +from flask import jsonify, request +from flask_cors import cross_origin + +from app.api.base import api_bp, verify_api_key +from app.config import EMAIL_DOMAIN, MAX_NB_EMAIL_FREE_PLAN +from app.extensions import db +from app.log import LOG +from app.models import GenEmail, AliasUsedOn +from app.utils import convert_to_id + + +@api_bp.route("/alias/random/new", methods=["POST"]) +@cross_origin() +@verify_api_key +def new_random_alias(): + """ + Create a new random alias + Output: + 201 if success + + """ + user = g.user + if not user.can_create_new_alias(): + LOG.d("user %s cannot create new random alias", user) + return ( + jsonify( + error=f"You have reached the limitation of a free account with the maximum of " + f"{MAX_NB_EMAIL_FREE_PLAN} aliases, please upgrade your plan to create more aliases" + ), + 400, + ) + + scheme = user.alias_generator + gen_email = GenEmail.create_new_random(user_id=user.id, scheme=scheme) + db.session.commit() + + hostname = request.args.get("hostname") + if hostname: + AliasUsedOn.create(gen_email_id=gen_email.id, hostname=hostname) + db.session.commit() + + return jsonify(alias=gen_email.email), 201 diff --git a/tests/api/test_new_random_alias.py b/tests/api/test_new_random_alias.py new file mode 100644 index 00000000..a50f3c1f --- /dev/null +++ b/tests/api/test_new_random_alias.py @@ -0,0 +1,52 @@ +from flask import url_for + +from app.config import EMAIL_DOMAIN, MAX_NB_EMAIL_FREE_PLAN +from app.extensions import db +from app.models import User, ApiKey, GenEmail + + +def test_success(flask_client): + user = User.create( + email="a@b.c", password="password", name="Test User", activated=True + ) + db.session.commit() + + # create api_key + api_key = ApiKey.create(user.id, "for test") + db.session.commit() + + r = flask_client.post( + url_for("api.new_random_alias", hostname="www.test.com"), + headers={"Authentication": api_key.code}, + ) + + assert r.status_code == 201 + assert r.json["alias"].endswith(EMAIL_DOMAIN) + + +def test_out_of_quota(flask_client): + user = User.create( + email="a@b.c", password="password", name="Test User", activated=True + ) + db.session.commit() + + # create api_key + api_key = ApiKey.create(user.id, "for test") + db.session.commit() + + # create 3 random alias to run out of quota + for _ in range(MAX_NB_EMAIL_FREE_PLAN): + GenEmail.create_new(user.id, prefix="test1") + GenEmail.create_new(user.id, prefix="test2") + GenEmail.create_new(user.id, prefix="test3") + + r = flask_client.post( + url_for("api.new_random_alias", hostname="www.test.com"), + headers={"Authentication": api_key.code}, + ) + + assert r.status_code == 400 + assert ( + r.json["error"] + == "You have reached the limitation of a free account with the maximum of 3 aliases, please upgrade your plan to create more aliases" + ) From f52f4c821b4428188d82042aa55d0af72f51018e Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 5 Jan 2020 22:48:38 +0100 Subject: [PATCH 59/76] Add /api/user_info --- README.md | 20 ++++++++++++++++++++ app/api/__init__.py | 2 +- app/api/views/user_info.py | 22 ++++++++++++++++++++++ tests/api/test_user_info.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 app/api/views/user_info.py create mode 100644 tests/api/test_user_info.py diff --git a/README.md b/README.md index a2c37df2..66e46207 100644 --- a/README.md +++ b/README.md @@ -442,6 +442,26 @@ Some errors should be fixed during development however: for example error like ` All following endpoint return `401` status code if the API Key is incorrect. +#### GET /api/user_info + +Given the API Key, return user name and whether user is premium. +This endpoint could be used to validate the api key. + +Input: +- `Authentication` header that contains the api key + +Output: if api key is correct, return a json with user name and whether user is premium, for example: + +```json +{ + "name": "John Wick", + "is_premium": false +} +``` + +If api key is incorrect, return 401. + + #### GET /api/v2/alias/options User alias info and suggestion. Used by the first extension screen when user opens the extension. diff --git a/app/api/__init__.py b/app/api/__init__.py index 612893b0..268191e9 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -1 +1 @@ -from .views import alias_options, new_custom_alias, new_random_alias +from .views import alias_options, new_custom_alias, new_random_alias, user_info diff --git a/app/api/views/user_info.py b/app/api/views/user_info.py new file mode 100644 index 00000000..e8a2dd94 --- /dev/null +++ b/app/api/views/user_info.py @@ -0,0 +1,22 @@ +from flask import jsonify, request, g +from flask_cors import cross_origin +from sqlalchemy import desc + +from app.api.base import api_bp, verify_api_key +from app.config import EMAIL_DOMAIN +from app.extensions import db +from app.log import LOG +from app.models import AliasUsedOn, GenEmail, User +from app.utils import convert_to_id, random_word + + +@api_bp.route("/user_info") +@cross_origin() +@verify_api_key +def user_info(): + """ + Return user info given the api-key + """ + user = g.user + + return jsonify({"name": user.name, "is_premium": user.is_premium()}) diff --git a/tests/api/test_user_info.py b/tests/api/test_user_info.py new file mode 100644 index 00000000..ffa96edf --- /dev/null +++ b/tests/api/test_user_info.py @@ -0,0 +1,32 @@ +from flask import url_for + +from app.extensions import db +from app.models import User, ApiKey, AliasUsedOn, GenEmail + + +def test_success(flask_client): + user = User.create( + email="a@b.c", password="password", name="Test User", activated=True + ) + db.session.commit() + + # create api_key + api_key = ApiKey.create(user.id, "for test") + db.session.commit() + + r = flask_client.get( + url_for("api.user_info"), headers={"Authentication": api_key.code} + ) + + assert r.status_code == 200 + assert r.json == {"is_premium": False, "name": "Test User"} + + +def test_wrong_api_key(flask_client): + r = flask_client.get( + url_for("api.user_info"), headers={"Authentication": "Invalid code"} + ) + + assert r.status_code == 401 + + assert r.json == {"error": "Wrong api key"} From 5af974fc5d4681e92319c58e00867008890c32f9 Mon Sep 17 00:00:00 2001 From: doanguyen Date: Sun, 5 Jan 2020 22:49:48 +0100 Subject: [PATCH 60/76] alias log dashboard --- app/config.py | 4 +- .../templates/dashboard/alias_log.html | 93 ++++++++++++++++++- app/dashboard/views/alias_log.py | 9 ++ 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/app/config.py b/app/config.py index 11608748..6da3176c 100644 --- a/app/config.py +++ b/app/config.py @@ -31,7 +31,7 @@ COLOR_LOG = "COLOR_LOG" in os.environ PROMO_CODE = "SIMPLEISBETTER" # Debug mode -DEBUG = os.environ['DEBUG'] +DEBUG = os.environ["DEBUG"] # Server url URL = os.environ["URL"] print(">>> URL:", URL) @@ -45,7 +45,7 @@ SUPPORT_EMAIL = os.environ["SUPPORT_EMAIL"] ADMIN_EMAIL = os.environ.get("ADMIN_EMAIL") MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"]) # allow to override postfix server locally -POSTFIX_SERVER = os.environ.get("POSTFIX_SERVER", "1.1.1.1") +POSTFIX_SERVER = os.environ.get("POSTFIX_SERVER", "0.0.0.0") # list of (priority, email server) EMAIL_SERVERS_WITH_PRIORITY = eval( diff --git a/app/dashboard/templates/dashboard/alias_log.html b/app/dashboard/templates/dashboard/alias_log.html index 59c40d06..2382d3c1 100644 --- a/app/dashboard/templates/dashboard/alias_log.html +++ b/app/dashboard/templates/dashboard/alias_log.html @@ -1,7 +1,68 @@ {% extends 'default.html' %} {% set active_page = "dashboard" %} +{% block head %} + +{% endblock %} {% block title %} Alias Activity {% endblock %} @@ -12,7 +73,37 @@ {{ alias }} - +
+
+
+ + {{ total }} + Email Handled +
+
+
+
+ + {{ email_forwarded }} + Email Forwarded +
+
+
+
+ + {{ email_replied }} + Email Replied +
+
+
+
+ + {{ email_forwarded }} + Email Blocked +
+
+
+

Activities

{% for log in logs %}
diff --git a/app/dashboard/views/alias_log.py b/app/dashboard/views/alias_log.py index d7ec6388..afc790f2 100644 --- a/app/dashboard/views/alias_log.py +++ b/app/dashboard/views/alias_log.py @@ -38,6 +38,15 @@ def alias_log(alias, page_id): return redirect(url_for("dashboard.index")) logs = get_alias_log(gen_email, page_id) + base = ( + db.session.query(ForwardEmail, ForwardEmailLog) + .filter(ForwardEmail.id == ForwardEmailLog.forward_id) + .filter(ForwardEmail.gen_email_id == gen_email.id) + ) + total = base.count() + email_forwarded = base.filter(ForwardEmailLog.is_reply == False).count() + email_replied = base.filter(ForwardEmailLog.is_reply == True).count() + email_blocked = base.filter(ForwardEmailLog.blocked == True).count() last_page = ( len(logs) < _LIMIT ) # lightweight pagination without counting all objects From 5ffdc45c8703d010c77d92edb4a9247a236626de Mon Sep 17 00:00:00 2001 From: doanguyen Date: Sun, 5 Jan 2020 22:53:00 +0100 Subject: [PATCH 61/76] fix DEBUG flag is not default in os environment --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 6da3176c..ace7ae05 100644 --- a/app/config.py +++ b/app/config.py @@ -31,7 +31,7 @@ COLOR_LOG = "COLOR_LOG" in os.environ PROMO_CODE = "SIMPLEISBETTER" # Debug mode -DEBUG = os.environ["DEBUG"] +DEBUG = os.environ["DEBUG"] if "DEBUG" in os.environ else False # Server url URL = os.environ["URL"] print(">>> URL:", URL) From 783aba12757dc3b00d6386390627d255d10d08e9 Mon Sep 17 00:00:00 2001 From: doanguyen Date: Sun, 5 Jan 2020 22:58:40 +0100 Subject: [PATCH 62/76] flask debug must be string, not bool, int. What a joke --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index 5425b926..16ff725a 100644 --- a/server.py +++ b/server.py @@ -69,7 +69,7 @@ def create_app() -> Flask: app = Flask(__name__) app.url_map.strict_slashes = False app.debug = DEBUG - os.environ["FLASK_DEBUG"] = DEBUG + os.environ["FLASK_DEBUG"] = str(DEBUG) app.config["SQLALCHEMY_DATABASE_URI"] = DB_URI app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False From 8f1c56baf989fcc60dcca4216adc4db6f7416d5c Mon Sep 17 00:00:00 2001 From: doanguyen Date: Sun, 5 Jan 2020 23:03:56 +0100 Subject: [PATCH 63/76] forget to push this local configuration --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index ace7ae05..3473ce8d 100644 --- a/app/config.py +++ b/app/config.py @@ -45,7 +45,7 @@ SUPPORT_EMAIL = os.environ["SUPPORT_EMAIL"] ADMIN_EMAIL = os.environ.get("ADMIN_EMAIL") MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"]) # allow to override postfix server locally -POSTFIX_SERVER = os.environ.get("POSTFIX_SERVER", "0.0.0.0") +POSTFIX_SERVER = os.environ.get("POSTFIX_SERVER", "1.1.1.1") # list of (priority, email server) EMAIL_SERVERS_WITH_PRIORITY = eval( From d527fcf64823d3ea696e3bd715d74d2973a3c9aa Mon Sep 17 00:00:00 2001 From: Son NK Date: Mon, 6 Jan 2020 16:11:17 +0100 Subject: [PATCH 64/76] Move "forgot password" button to a different position to avoid Keepass issue --- app/auth/templates/auth/login.html | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/app/auth/templates/auth/login.html b/app/auth/templates/auth/login.html index fb8fa783..e584ba5c 100644 --- a/app/auth/templates/auth/login.html +++ b/app/auth/templates/auth/login.html @@ -39,32 +39,26 @@
{{ form.password(class="form-control", type="password") }} {{ render_field_errors(form.password) }} +
- - + +
+ Don't have an account yet? Sign up +
+
- -
- Don't have an account yet? Sign up -
From ca37ce5e5a681b148f73eb1059e8717939cee026 Mon Sep 17 00:00:00 2001 From: Son NK Date: Mon, 6 Jan 2020 19:41:05 +0100 Subject: [PATCH 65/76] add id to notification section in setting --- app/dashboard/templates/dashboard/setting.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dashboard/templates/dashboard/setting.html b/app/dashboard/templates/dashboard/setting.html index eac8c317..49d6d37d 100644 --- a/app/dashboard/templates/dashboard/setting.html +++ b/app/dashboard/templates/dashboard/setting.html @@ -90,7 +90,7 @@
-

Notifications

+

Notifications

Do you want to receive our newsletter?
From 6a99fd30c4cd14c089f9468b97e670142fc098af Mon Sep 17 00:00:00 2001 From: doanguyen Date: Mon, 6 Jan 2020 23:58:24 +0100 Subject: [PATCH 66/76] fix some minor bugs --- app/dashboard/templates/dashboard/alias_log.html | 2 +- app/dashboard/views/alias_log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dashboard/templates/dashboard/alias_log.html b/app/dashboard/templates/dashboard/alias_log.html index 2382d3c1..130a37b5 100644 --- a/app/dashboard/templates/dashboard/alias_log.html +++ b/app/dashboard/templates/dashboard/alias_log.html @@ -98,7 +98,7 @@
- {{ email_forwarded }} + {{ email_blocked }} Email Blocked
diff --git a/app/dashboard/views/alias_log.py b/app/dashboard/views/alias_log.py index afc790f2..12221ded 100644 --- a/app/dashboard/views/alias_log.py +++ b/app/dashboard/views/alias_log.py @@ -44,7 +44,7 @@ def alias_log(alias, page_id): .filter(ForwardEmail.gen_email_id == gen_email.id) ) total = base.count() - email_forwarded = base.filter(ForwardEmailLog.is_reply == False).count() + email_forwarded = base.filter(ForwardEmailLog.is_reply == False).filter(ForwardEmailLog.blocked==False).count() email_replied = base.filter(ForwardEmailLog.is_reply == True).count() email_blocked = base.filter(ForwardEmailLog.blocked == True).count() last_page = ( From d804a28c071a204ca3640e7f99aca4a57f858469 Mon Sep 17 00:00:00 2001 From: doanguyen Date: Tue, 7 Jan 2020 00:02:12 +0100 Subject: [PATCH 67/76] fix the format, again --- app/dashboard/views/alias_log.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/dashboard/views/alias_log.py b/app/dashboard/views/alias_log.py index 12221ded..6878b130 100644 --- a/app/dashboard/views/alias_log.py +++ b/app/dashboard/views/alias_log.py @@ -44,7 +44,11 @@ def alias_log(alias, page_id): .filter(ForwardEmail.gen_email_id == gen_email.id) ) total = base.count() - email_forwarded = base.filter(ForwardEmailLog.is_reply == False).filter(ForwardEmailLog.blocked==False).count() + email_forwarded = ( + base.filter(ForwardEmailLog.is_reply == False) + .filter(ForwardEmailLog.blocked == False) + .count() + ) email_replied = base.filter(ForwardEmailLog.is_reply == True).count() email_blocked = base.filter(ForwardEmailLog.blocked == True).count() last_page = ( From 072e73187b3bba84a5da5e5d8710426d0475f8a8 Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 19:14:36 +0100 Subject: [PATCH 68/76] make sure to replace Reply-To header in reply phase --- email_handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 22a16a4d..38aa972e 100644 --- a/email_handler.py +++ b/email_handler.py @@ -251,8 +251,13 @@ class MailHandler: LOG.d("Remove DKIM-Signature %s", msg["DKIM-Signature"]) del msg["DKIM-Signature"] - # email seems to come from alias + # the email comes from alias msg.replace_header("From", alias) + + # some email providers like ProtonMail adds automatically the Reply-To field + # make sure to replace it too + add_or_replace_header(msg, "Reply-To", alias) + msg.replace_header("To", forward_email.website_email) # add List-Unsubscribe header From 3bca9fde6ba98d3f14f44969589f6ef016980a9a Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 19:46:57 +0100 Subject: [PATCH 69/76] refactor: move add_or_replace_header to email_utils --- app/email_utils.py | 10 +++++++++- email_handler.py | 10 +--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index bc12eb82..60673116 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1,5 +1,5 @@ import os -from email.message import EmailMessage +from email.message import EmailMessage, Message from email.utils import make_msgid, formatdate from smtplib import SMTP @@ -197,3 +197,11 @@ def add_dkim_signature(msg: EmailMessage, email_domain: str): sig = sig.replace("\n", " ").replace("\r", "") msg.add_header("DKIM-Signature", sig[len("DKIM-Signature: ") :]) + + +def add_or_replace_header(msg: Message, header: str, value: str): + try: + msg.add_header(header, value) + except ValueError: + # the header exists already + msg.replace_header(header, value) \ No newline at end of file diff --git a/email_handler.py b/email_handler.py index 38aa972e..60572a27 100644 --- a/email_handler.py +++ b/email_handler.py @@ -45,7 +45,7 @@ from app.email_utils import ( send_email, add_dkim_signature, get_email_domain_part, -) + add_or_replace_header) from app.extensions import db from app.log import LOG from app.models import GenEmail, ForwardEmail, ForwardEmailLog, CustomDomain @@ -298,14 +298,6 @@ class MailHandler: return "250 Message accepted for delivery" -def add_or_replace_header(msg: Message, header: str, value: str): - try: - msg.add_header(header, value) - except ValueError: - # the header exists already - msg.replace_header(header, value) - - if __name__ == "__main__": controller = Controller(MailHandler(), hostname="0.0.0.0", port=20381) From 44527c6c4ed6d58d5c7dc21f3ef0b79956e2eaa6 Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 19:47:26 +0100 Subject: [PATCH 70/76] fix annotation on email_utils --- app/email_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/email_utils.py b/app/email_utils.py index 60673116..2ae3cb89 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -177,7 +177,7 @@ def get_email_domain_part(email): return email[email.find("@") + 1 :] -def add_dkim_signature(msg: EmailMessage, email_domain: str): +def add_dkim_signature(msg: Message, email_domain: str): if msg["DKIM-Signature"]: LOG.d("Remove DKIM-Signature %s", msg["DKIM-Signature"]) del msg["DKIM-Signature"] From ba46d8f7e06d73c73de1851c1642f0efdb10f0d0 Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 19:49:26 +0100 Subject: [PATCH 71/76] add delete_header() --- app/email_utils.py | 7 ++++++- email_handler.py | 11 +++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 2ae3cb89..a07c690d 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -204,4 +204,9 @@ def add_or_replace_header(msg: Message, header: str, value: str): msg.add_header(header, value) except ValueError: # the header exists already - msg.replace_header(header, value) \ No newline at end of file + msg.replace_header(header, value) + + +def delete_header(msg: Message, header: str): + if msg[header]: + del msg[header] \ No newline at end of file diff --git a/email_handler.py b/email_handler.py index 60572a27..983e6fb2 100644 --- a/email_handler.py +++ b/email_handler.py @@ -45,7 +45,7 @@ from app.email_utils import ( send_email, add_dkim_signature, get_email_domain_part, - add_or_replace_header) + add_or_replace_header, delete_header) from app.extensions import db from app.log import LOG from app.models import GenEmail, ForwardEmail, ForwardEmailLog, CustomDomain @@ -162,9 +162,7 @@ class MailHandler: add_or_replace_header(msg, "X-SimpleLogin-Type", "Forward") # remove reply-to header if present - if msg["Reply-To"]: - LOG.d("Delete reply-to header %s", msg["Reply-To"]) - del msg["Reply-To"] + delete_header(msg, "Reply-To") # change the from header so the sender comes from @SL # so it can pass DMARC check @@ -246,10 +244,7 @@ class MailHandler: return "450 ignored" - # remove DKIM-Signature - if msg["DKIM-Signature"]: - LOG.d("Remove DKIM-Signature %s", msg["DKIM-Signature"]) - del msg["DKIM-Signature"] + delete_header(msg, "DKIM-Signature") # the email comes from alias msg.replace_header("From", alias) From f1befd70244dda83f85b3f6e461ae29e3eb94003 Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 19:50:24 +0100 Subject: [PATCH 72/76] delete Reply-To header in reply phase --- email_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/email_handler.py b/email_handler.py index 983e6fb2..9a69281d 100644 --- a/email_handler.py +++ b/email_handler.py @@ -250,8 +250,8 @@ class MailHandler: msg.replace_header("From", alias) # some email providers like ProtonMail adds automatically the Reply-To field - # make sure to replace it too - add_or_replace_header(msg, "Reply-To", alias) + # make sure to delete it + delete_header(msg, "Reply-To") msg.replace_header("To", forward_email.website_email) From 27b9312057f44eb7a11b465b05870f001c6019b2 Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 19:50:36 +0100 Subject: [PATCH 73/76] fix formatting --- app/email_utils.py | 2 +- email_handler.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index a07c690d..895a5614 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -209,4 +209,4 @@ def add_or_replace_header(msg: Message, header: str, value: str): def delete_header(msg: Message, header: str): if msg[header]: - del msg[header] \ No newline at end of file + del msg[header] diff --git a/email_handler.py b/email_handler.py index 9a69281d..20ff8e3c 100644 --- a/email_handler.py +++ b/email_handler.py @@ -45,7 +45,9 @@ from app.email_utils import ( send_email, add_dkim_signature, get_email_domain_part, - add_or_replace_header, delete_header) + add_or_replace_header, + delete_header, +) from app.extensions import db from app.log import LOG from app.models import GenEmail, ForwardEmail, ForwardEmailLog, CustomDomain From aa10cdb3ee0b346ef6f6a0991203c3e4595ed208 Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 21:53:00 +0100 Subject: [PATCH 74/76] =?UTF-8?q?If=20domain=20is=20not=20verified,=20clic?= =?UTF-8?q?king=20on=20=F0=9F=9A=AB=20brings=20user=20to=20DNS=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/dashboard/templates/dashboard/custom_domain.html | 6 +++++- app/dashboard/templates/dashboard/domain_detail/info.html | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/dashboard/templates/dashboard/custom_domain.html b/app/dashboard/templates/dashboard/custom_domain.html index 0f448388..6423fe44 100644 --- a/app/dashboard/templates/dashboard/custom_domain.html +++ b/app/dashboard/templates/dashboard/custom_domain.html @@ -21,7 +21,11 @@ {% if custom_domain.verified %} {% else %} - 🚫 + + 🚫 + + {% endif %}
diff --git a/app/dashboard/templates/dashboard/domain_detail/info.html b/app/dashboard/templates/dashboard/domain_detail/info.html index 4315d790..f208e87f 100644 --- a/app/dashboard/templates/dashboard/domain_detail/info.html +++ b/app/dashboard/templates/dashboard/domain_detail/info.html @@ -11,7 +11,11 @@ {% if custom_domain.verified %} {% else %} - 🚫 + + 🚫 + + {% endif %}
From d9f2ec214f72c87e323dd36c78aaa75a267bebbe Mon Sep 17 00:00:00 2001 From: Son NK Date: Tue, 7 Jan 2020 22:13:12 +0100 Subject: [PATCH 75/76] add @ warning when setup DNS --- app/dashboard/templates/dashboard/domain_detail/dns.html | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/dns.html b/app/dashboard/templates/dashboard/domain_detail/dns.html index e61c9cb9..cf1ffd43 100644 --- a/app/dashboard/templates/dashboard/domain_detail/dns.html +++ b/app/dashboard/templates/dashboard/domain_detail/dns.html @@ -27,13 +27,14 @@ {% endif %} -
Add the following MX DNS record to your domain. - Please note that there's a point (.) at the end of target addresses. +
Add the following MX DNS record to your domain.
+ Please note that there's a point (.) at the end target addresses.
+ Also some domain registrars (Namecheap, CloudFlare, etc) might use @ for the root domain.
{% for priority, email_server in EMAIL_SERVERS_WITH_PRIORITY %}
- Domain: {{ custom_domain.domain }}
+ Domain: {{ custom_domain.domain }} or @
Priority: {{ priority }}
Target: {{ email_server }}
@@ -91,7 +92,7 @@
Add the following TXT DNS record to your domain
- Domain: {{ custom_domain.domain }}
+ Domain: {{ custom_domain.domain }} or @
Value: {{ spf_record }} From c49bc87baef0d09555abf52c8503253acf170584 Mon Sep 17 00:00:00 2001 From: doanguyen Date: Tue, 7 Jan 2020 22:29:37 +0100 Subject: [PATCH 76/76] rollback the debug flag --- server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server.py b/server.py index 16ff725a..e51272c2 100644 --- a/server.py +++ b/server.py @@ -68,8 +68,6 @@ os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" def create_app() -> Flask: app = Flask(__name__) app.url_map.strict_slashes = False - app.debug = DEBUG - os.environ["FLASK_DEBUG"] = str(DEBUG) app.config["SQLALCHEMY_DATABASE_URI"] = DB_URI app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False @@ -445,6 +443,6 @@ if __name__ == "__main__": context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) context.load_cert_chain("local_data/cert.pem", "local_data/key.pem") - app.run(host="0.0.0.0", port=7777, ssl_context=context) + app.run(debug=True, host="0.0.0.0", port=7777, ssl_context=context) else: - app.run(host="0.0.0.0", port=7777) + app.run(debug=True, host="0.0.0.0", port=7777)