From 1384ccc4594c45b80d857b7b5b4da9082a813584 Mon Sep 17 00:00:00 2001 From: Son Date: Sun, 15 Aug 2021 17:32:33 +0200 Subject: [PATCH 001/144] remove RESET_DB config --- app/config.py | 1 - example.env | 3 --- server.py | 10 ---------- tests/test.env | 2 -- 4 files changed, 16 deletions(-) diff --git a/app/config.py b/app/config.py index 432fa075..c49109e8 100644 --- a/app/config.py +++ b/app/config.py @@ -45,7 +45,6 @@ if config_file: else: load_dotenv() -RESET_DB = "RESET_DB" in os.environ COLOR_LOG = "COLOR_LOG" in os.environ # Allow user to have 1 year of premium: set the expiration_date to 1 year more diff --git a/example.env b/example.env index 2e7e859f..5b00468e 100644 --- a/example.env +++ b/example.env @@ -74,9 +74,6 @@ EMAIL_SERVERS_WITH_PRIORITY=[(10, "email.hostname.")] # the DKIM private key used to compute DKIM-Signature # DKIM_PRIVATE_KEY_PATH=local_data/dkim.key -# delete and recreate the sqlite database, for local development -RESET_DB=true - # DB Connection # Local SQLite database DB_URI=sqlite:///db.sqlite diff --git a/server.py b/server.py index 15bda4b9..d0c60f51 100644 --- a/server.py +++ b/server.py @@ -49,7 +49,6 @@ from app.config import ( URL, SHA1, PADDLE_MONTHLY_PRODUCT_ID, - RESET_DB, FLASK_PROFILER_PATH, FLASK_PROFILER_PASSWORD, SENTRY_FRONT_END_DSN, @@ -951,15 +950,6 @@ def local_main(): app.debug = True DebugToolbarExtension(app) - # warning: only used in local - if RESET_DB: - from init_app import add_sl_domains - - LOG.warning("reset db, add fake data") - with app.app_context(): - fake_data() - add_sl_domains() - app.run(debug=True, port=7777) # uncomment to run https locally diff --git a/tests/test.env b/tests/test.env index 8b5669fd..3c5b3ff9 100644 --- a/tests/test.env +++ b/tests/test.env @@ -14,8 +14,6 @@ MAX_NB_EMAIL_FREE_PLAN=3 EMAIL_SERVERS_WITH_PRIORITY=[(10, "email.hostname.")] DKIM_PRIVATE_KEY_PATH=local_data/dkim.key -# Database -RESET_DB=true DB_URI=postgresql://test:test@localhost:5432/test # Flask From 4cbbf260d489e564fb729f7955d748fa384e82f0 Mon Sep 17 00:00:00 2001 From: Son Date: Sun, 15 Aug 2021 17:32:54 +0200 Subject: [PATCH 002/144] add dummy-data flask command --- server.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server.py b/server.py index d0c60f51..36a44b9a 100644 --- a/server.py +++ b/server.py @@ -204,13 +204,7 @@ def create_app() -> Flask: def fake_data(): LOG.d("create fake data") - # Remove db if exist - if os.path.exists("db.sqlite"): - LOG.d("remove existing db file") - os.remove("db.sqlite") - # Create all tables - db.create_all() # Create a user user = User.create( @@ -918,6 +912,15 @@ def register_custom_commands(app): LOG.d("finish trunk %s, update %s email logs", trunk, nb_update) db.session.commit() + @app.cli.command("dummy-data") + def dummy_data(): + from init_app import add_sl_domains + + LOG.warning("reset db, add fake data") + with app.app_context(): + fake_data() + add_sl_domains() + def setup_do_not_track(app): @app.route("/dnt") From d9c682a23e1d78243889091a23992acd14f82ca7 Mon Sep 17 00:00:00 2001 From: Son Date: Sun, 15 Aug 2021 17:41:16 +0200 Subject: [PATCH 003/144] remove sqlite everywhere, only use postgres. Do not use 5432 port to avoid conflict --- .github/workflows/main.yml | 5 ++-- CONTRIBUTING.md | 50 ++++++++++++++++---------------------- example.env | 5 +--- new_migration.sh | 6 ++--- tests/test.env | 2 +- 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 58ccb1bb..5256b69e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,12 +25,11 @@ jobs: # required POSTGRES_PASSWORD: test # optional (defaults to `5432`) - POSTGRES_PORT: 5432 + POSTGRES_PORT: 15432 # optional (defaults to `postgres`) POSTGRES_USER: test ports: - # maps tcp port 5432 on service container to the host - - 5432:5432 + - 15432:15432 # set health checks to wait until postgres has started options: >- --health-cmd pg_isready diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8e8243ff..ace9b5ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,6 @@ Thanks for taking the time to contribute! 🎉👍 -The project uses Flask and requires Python3.7+. - -## Quick start - -If you have Docker installed, run the following command to start SimpleLogin local server: - - -```bash -docker run --name sl -it --rm \ - -e RESET_DB=true \ - -e CONFIG=/code/example.env \ - -p 7777:7777 \ - simplelogin/app:3.4.0 python server.py -``` - -Then open http://localhost:7777, you should be able to login with `john@wick.com/password` account. +The project uses Flask, Python3.7+ and requires Postgres 12+ as dependency. ## General Architecture @@ -25,15 +10,16 @@ Then open http://localhost:7777, you should be able to login with `john@wick.com SimpleLogin backend consists of 2 main components: -- the `webapp` used by several clients: web UI (the dashboard), browser extension (Chrome & Firefox for now), OAuth clients (apps that integrate "Login with SimpleLogin" button) and mobile app (work in progress). +- the `webapp` used by several clients: the web app, the browser extensions (Chrome & Firefox for now), OAuth clients (apps that integrate "Sign in with SimpleLogin" button) and mobile apps. - the `email handler`: implements the email forwarding (i.e. alias receiving email) and email sending (i.e. alias sending email). -## Run code locally +## Install dependencies -The project uses +The project requires: - Python 3.7+ and [poetry](https://python-poetry.org/) to manage dependencies - Node v10 for front-end. +- Postgres 12+ First, install all dependencies by running the following command. Feel free to use `virtualenv` or similar tools to isolate development environment. @@ -42,22 +28,24 @@ Feel free to use `virtualenv` or similar tools to isolate development environmen poetry install ``` -On Mac, sometimes you might need to install some other packages like +On Mac, sometimes you might need to install some other packages via `brew`: ```bash brew install pkg-config libffi openssl postgresql ``` -You also need to install `gpg`, on Mac it can be done with: +You also need to install `gpg` tool, on Mac it can be done with: ```bash brew install gnupg ``` -Then make sure all tests pass. You need to run a local postgres database to run tests, it can be run with docker with: +## Run tests + +Running test requires a Postgres database. You can run one with docker: ```bash -docker run -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test -p 5432:5432 postgres:13 +docker run -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test -p 15432:5432 postgres:13 ``` then run all tests @@ -66,6 +54,8 @@ then run all tests pytest ``` +## Run the code locally + Install npm packages ```bash @@ -78,17 +68,19 @@ To run the code locally, please create a local setting file based on `example.en cp example.env .env ``` +Run the postgres database: + +```bash +docker run -e POSTGRES_PASSWORD=mypassword -e POSTGRES_USER=myuser -e POSTGRES_DB=simplelogin -p 35432:5432 postgres:13 +``` + To run the server: ``` -python3 server.py +flask db upgrade && flask dummy-data && python3 server.py ``` -then open http://localhost:7777, you should be able to login with the following account - -``` -john@wick.com / password -``` +then open http://localhost:7777, you should be able to login with `john@wick.com / password` account. You might need to change the `.env` file for developing certain features. This file is ignored by git. diff --git a/example.env b/example.env index 5b00468e..9dfac231 100644 --- a/example.env +++ b/example.env @@ -75,10 +75,7 @@ EMAIL_SERVERS_WITH_PRIORITY=[(10, "email.hostname.")] # DKIM_PRIVATE_KEY_PATH=local_data/dkim.key # DB Connection -# Local SQLite database -DB_URI=sqlite:///db.sqlite -# Postgres -# DB_URI=postgresql://myuser:mypassword@sl-db:5432/simplelogin +DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin FLASK_SECRET=secret diff --git a/new_migration.sh b/new_migration.sh index 660ccd17..476a7d87 100644 --- a/new_migration.sh +++ b/new_migration.sh @@ -4,13 +4,13 @@ # create a postgres database for SimpleLogin docker rm -f sl-db -docker run -p 15432:5432 --name sl-db -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=sl -d postgres +docker run -p 25432:5432 --name sl-db -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=sl -d postgres:13 # run run `flask db upgrade` to upgrade the DB to the latest stage and -env DB_URI=postgresql://postgres:postgres@127.0.0.1:15432/sl flask db upgrade +env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl flask db upgrade # finally `flask db migrate` to generate the migration script. -env DB_URI=postgresql://postgres:postgres@127.0.0.1:15432/sl flask db migrate +env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl flask db migrate # remove the db docker rm -f sl-db \ No newline at end of file diff --git a/tests/test.env b/tests/test.env index 3c5b3ff9..d1fbe886 100644 --- a/tests/test.env +++ b/tests/test.env @@ -14,7 +14,7 @@ MAX_NB_EMAIL_FREE_PLAN=3 EMAIL_SERVERS_WITH_PRIORITY=[(10, "email.hostname.")] DKIM_PRIVATE_KEY_PATH=local_data/dkim.key -DB_URI=postgresql://test:test@localhost:5432/test +DB_URI=postgresql://test:test@localhost:15432/test # Flask FLASK_SECRET=secret From 66bafe7439b277a2cab61db2a05a1e0bf7b8f727 Mon Sep 17 00:00:00 2001 From: Son Date: Sun, 15 Aug 2021 17:42:15 +0200 Subject: [PATCH 004/144] flake8 --- server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server.py b/server.py index 36a44b9a..cb99cd57 100644 --- a/server.py +++ b/server.py @@ -205,7 +205,6 @@ def create_app() -> Flask: def fake_data(): LOG.d("create fake data") - # Create a user user = User.create( email="john@wick.com", From e5308932a2d013485553b5b7e87eb07cf3ad28a0 Mon Sep 17 00:00:00 2001 From: Son Date: Sun, 15 Aug 2021 17:50:47 +0200 Subject: [PATCH 005/144] make mailbox deletion async --- app/dashboard/views/domain_detail.py | 2 +- app/dashboard/views/mailbox.py | 39 +++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index 6d65bea6..9f6eb746 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -208,7 +208,7 @@ def domain_detail(custom_domain_id): return render_template("dashboard/domain_detail/info.html", **locals()) -def delete_domain(custom_domain_id: CustomDomain): +def delete_domain(custom_domain_id: int): from server import create_light_app with create_light_app().app_context(): diff --git a/app/dashboard/views/mailbox.py b/app/dashboard/views/mailbox.py index c3ee81ac..42774014 100644 --- a/app/dashboard/views/mailbox.py +++ b/app/dashboard/views/mailbox.py @@ -1,3 +1,5 @@ +from threading import Thread + from flask import render_template, request, redirect, url_for, flash from flask_login import login_required, current_user from flask_wtf import FlaskForm @@ -49,10 +51,13 @@ def mailbox_route(): flash("You cannot delete default mailbox", "error") return redirect(url_for("dashboard.mailbox_route")) - email = mailbox.email - Mailbox.delete(mailbox_id) - db.session.commit() - flash(f"Mailbox {email} has been deleted", "success") + LOG.d("Schedule deleting %s", mailbox) + Thread(target=delete_mailbox, args=(mailbox.id,)).start() + flash( + f"Mailbox {mailbox.email} scheduled for deletion." + f"You will receive a confirmation email when the deletion is finished", + "success", + ) return redirect(url_for("dashboard.mailbox_route")) if request.form.get("form-name") == "set-default": @@ -119,6 +124,32 @@ def mailbox_route(): ) +def delete_mailbox(mailbox_id: int): + from server import create_light_app + + with create_light_app().app_context(): + mailbox = Mailbox.get(mailbox_id) + if not mailbox: + return + + mailbox_email = mailbox.email + user = mailbox.user + + Mailbox.delete(mailbox_id) + db.session.commit() + LOG.d("Mailbox %s %s deleted", mailbox_id,mailbox_email) + + send_email( + user.email, + f"Your mailbox {mailbox_email} has been deleted", + f"""Mailbox {mailbox_email} along with its aliases are deleted successfully. + +Regards, +SimpleLogin team. + """, + ) + + def send_verification_email(user, mailbox): s = Signer(MAILBOX_SECRET) mailbox_id_signed = s.sign(str(mailbox.id)).decode() From 0f4ad1a0d453ecdd266c0e4605db508c471dc065 Mon Sep 17 00:00:00 2001 From: Son Date: Sun, 15 Aug 2021 17:56:31 +0200 Subject: [PATCH 006/144] black --- app/dashboard/views/mailbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dashboard/views/mailbox.py b/app/dashboard/views/mailbox.py index 42774014..2f6fd867 100644 --- a/app/dashboard/views/mailbox.py +++ b/app/dashboard/views/mailbox.py @@ -137,7 +137,7 @@ def delete_mailbox(mailbox_id: int): Mailbox.delete(mailbox_id) db.session.commit() - LOG.d("Mailbox %s %s deleted", mailbox_id,mailbox_email) + LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email) send_email( user.email, From 1678945d5a7212465169231b837b2adead8cb910 Mon Sep 17 00:00:00 2001 From: Son Date: Sun, 15 Aug 2021 17:58:49 +0200 Subject: [PATCH 007/144] improve wording --- app/dashboard/templates/dashboard/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index 731c2a79..de8510ea 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -384,7 +384,7 @@ {% endif %} -
Alias Note
+
Note
+
+ + +
+
@@ -390,25 +409,6 @@
{% endif %} -
Note
-
-
- -
- - -
-
Display name diff --git a/static/js/index.js b/static/js/index.js index 4e8a8e53..dab6b406 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -172,7 +172,7 @@ $(".save-note").on("click", async function () { }); if (res.ok) { - toastr.success(`Note Saved`); + toastr.success(`Saved`); } else { toastr.error("Sorry for the inconvenience! Could you refresh the page & retry please?", "Unknown Error"); // reset to the original value From 40915ad74140b7f42f18af6f4bec69fffd86a510 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sat, 21 Aug 2021 16:18:01 +0200 Subject: [PATCH 029/144] make alias description font smaller --- app/dashboard/templates/dashboard/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index 2d3c0ac4..6cc04271 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -320,6 +320,7 @@ id="note-{{ alias.id }}" name="note" class="form-control" + style="font-size: 12px" rows="2" placeholder="e.g. where the alias is used or why is it created">{{ alias.note or "" }}
From ef6388887f110d54f943f518aa674136299010f0 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sat, 21 Aug 2021 17:56:09 +0200 Subject: [PATCH 030/144] better filter app --- app/dashboard/templates/dashboard/index.html | 200 ++++++++++--------- 1 file changed, 105 insertions(+), 95 deletions(-) diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index 6cc04271..1c878d84 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -92,124 +92,133 @@
- -
- -
-
-
- - -
-
-
- - -
- -
-
+ +
+
+
+
+
+
+ + +
+
- - -
-
-
-
- - - +
+ +
+
+
+ + + +
+
+
+
+ + + +
+
+
+ +
-
-
+
+ +
+
+ - -
- - + + + + + {% for mailbox in current_user.mailboxes() %} + - - - - - + {% endfor %} + - + - - {% if query or sort or filter %} - Reset - {% endif %} - -
+ -
- - Filters - +
+ + {% if query or sort or filter %} + Reset + {% endif %} - +
+ +
{% for alias_info in alias_infos %} {% set alias = alias_info.alias %} @@ -502,6 +511,7 @@
{% endfor %}
+ {% if page > 0 or not last_page %} From 8b86851530ed07a86ee197688d8fe1de589fbfa8 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sat, 21 Aug 2021 19:03:48 +0200 Subject: [PATCH 031/144] Use AGPL license instead of MIT --- LICENSE | 674 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 657 insertions(+), 17 deletions(-) diff --git a/LICENSE b/LICENSE index 7e1ae8fa..0ad25db4 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,661 @@ -MIT License + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 -Copyright (c) 2020 SimpleLogin + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Preamble -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. From 3faf5c921d0c4207f2d0b059d886e4a50a24d47f Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 24 Aug 2021 19:43:32 +0200 Subject: [PATCH 032/144] add admin for custom domain --- app/admin_model.py | 6 ++++++ server.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/app/admin_model.py b/app/admin_model.py index ddf42b83..17c0e01e 100644 --- a/app/admin_model.py +++ b/app/admin_model.py @@ -224,6 +224,12 @@ class ClientAdmin(SLModelView): can_edit = True +class CustomDomainAdmin(SLModelView): + column_searchable_list = ["domain", "user.email", "user.id"] + column_exclude_list = ["ownership_txt_token"] + can_edit = False + + class ReferralAdmin(SLModelView): column_searchable_list = ["id", "user.email", "code", "name"] column_filters = ["id", "user.email", "code", "name"] diff --git a/server.py b/server.py index c1747127..8726bb20 100644 --- a/server.py +++ b/server.py @@ -39,6 +39,7 @@ from app.admin_model import ( ReferralAdmin, PayoutAdmin, CouponAdmin, + CustomDomainAdmin, ) from app.api.base import api_bp from app.auth.base import auth_bp @@ -883,6 +884,7 @@ def init_admin(app): admin.add_view(CouponAdmin(Coupon, db.session)) admin.add_view(ManualSubscriptionAdmin(ManualSubscription, db.session)) admin.add_view(ClientAdmin(Client, db.session)) + admin.add_view(CustomDomainAdmin(CustomDomain, db.session)) admin.add_view(ReferralAdmin(Referral, db.session)) admin.add_view(PayoutAdmin(Payout, db.session)) From ffd2ec5e81c1a38086240d20f30aafd0b45bd1cd Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sun, 5 Sep 2021 19:11:05 +0200 Subject: [PATCH 033/144] add filter by directory on dashboard --- app/api/serializer.py | 11 ++++++++++- app/dashboard/templates/dashboard/index.html | 8 ++++++++ app/dashboard/views/index.py | 6 +++++- app/models.py | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/app/api/serializer.py b/app/api/serializer.py index 2b9e2657..91849a46 100644 --- a/app/api/serializer.py +++ b/app/api/serializer.py @@ -129,7 +129,13 @@ def get_alias_infos_with_pagination(user, page_id=0, query=None) -> [AliasInfo]: def get_alias_infos_with_pagination_v3( - user, page_id=0, query=None, sort=None, alias_filter=None, mailbox_id=None + user, + page_id=0, + query=None, + sort=None, + alias_filter=None, + mailbox_id=None, + directory_id=None, ) -> [AliasInfo]: # subquery on alias annotated with nb_reply, nb_blocked, nb_forward, max_created_at, latest_email_log_created_at alias_activity_subquery = ( @@ -232,6 +238,9 @@ def get_alias_infos_with_pagination_v3( or_(Alias.mailbox_id == mailbox_id, AliasMailbox.mailbox_id == mailbox_id) ) + if directory_id: + q = q.filter(Alias.directory_id == directory_id) + if alias_filter == "enabled": q = q.filter(Alias.enabled) elif alias_filter == "disabled": diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index 1c878d84..2cc67e1f 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -192,6 +192,14 @@ {{ mailbox.email }}'s aliases {% endfor %} + + {% for directory in current_user.directories %} + + {% endfor %} + Date: Mon, 6 Sep 2021 18:51:50 +0200 Subject: [PATCH 034/144] use parsleyjs instead of formbouncerjs --- .../templates/dashboard/custom_alias.html | 10 ++--- app/oauth/templates/oauth/authorize.html | 20 +++------- static/package-lock.json | 13 ++++--- static/package.json | 2 +- static/style.css | 39 ++++++++++++++++++- templates/base.html | 5 ++- 6 files changed, 59 insertions(+), 30 deletions(-) diff --git a/app/dashboard/templates/dashboard/custom_alias.html b/app/dashboard/templates/dashboard/custom_alias.html index bd2ab982..bca4fe5a 100644 --- a/app/dashboard/templates/dashboard/custom_alias.html +++ b/app/dashboard/templates/dashboard/custom_alias.html @@ -25,15 +25,16 @@
{% endif %} -
+
@@ -105,9 +106,6 @@ {% block script %} -{% endblock %} \ No newline at end of file diff --git a/static/package-lock.json b/static/package-lock.json index 13aa66a7..6e7b2ef1 100644 --- a/static/package-lock.json +++ b/static/package-lock.json @@ -81,11 +81,6 @@ "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.7.0.tgz", "integrity": "sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM=" }, - "formbouncerjs": { - "version": "1.4.6", - "resolved": "https://registry.npmjs.org/formbouncerjs/-/formbouncerjs-1.4.6.tgz", - "integrity": "sha512-uQL/ZFs5w79RPtgtPbmL8HTKrsEK2q7Fph/uBqe4sFNXIszcrVmHcJ3aj5sB4OTsFbV6igQX9819f/f5t5Wjgg==" - }, "intro.js": { "version": "2.9.3", "resolved": "https://registry.npmjs.org/intro.js/-/intro.js-2.9.3.tgz", @@ -101,6 +96,14 @@ "resolved": "https://registry.npmjs.org/multiple-select/-/multiple-select-1.5.2.tgz", "integrity": "sha512-sTNNRrjnTtB1b1+HTKcjQ/mjWY7Gvigo9F3C/3oTQCTFEpYzwaRYFPRAOu2SogfA1hEfyJTXjyS1VAbanJMsmA==" }, + "parsleyjs": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/parsleyjs/-/parsleyjs-2.9.2.tgz", + "integrity": "sha512-DKS2XXTjEUZ1BJWUzgXAr+550kFBZrom2WYweubqdV7WzdNC1hjOajZDfeBPoAZMkXumJPlB3v37IKatbiW8zQ==", + "requires": { + "jquery": ">=1.8.0" + } + }, "popper.js": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz", diff --git a/static/package.json b/static/package.json index e9f7cb48..6342fb98 100644 --- a/static/package.json +++ b/static/package.json @@ -19,9 +19,9 @@ "@sentry/browser": "^5.30.0", "bootbox": "^5.5.2", "font-awesome": "^4.7.0", - "formbouncerjs": "^1.4.6", "intro.js": "^2.9.3", "multiple-select": "^1.5.2", + "parsleyjs": "^2.9.2", "qrious": "^4.0.2", "toastr": "^2.1.4", "vue": "^2.6.14" diff --git a/static/style.css b/static/style.css index 5ec0300d..d8f0c233 100644 --- a/static/style.css +++ b/static/style.css @@ -128,4 +128,41 @@ em { .disabled-content { pointer-events: none; opacity: 0.4; -} \ No newline at end of file +} + +/* Parsley CSS */ +input.parsley-success, +select.parsley-success, +textarea.parsley-success { + color: #468847; + background-color: #DFF0D8; + border: 1px solid #D6E9C6; +} + +input.parsley-error, +select.parsley-error, +textarea.parsley-error { + color: #B94A48; + background-color: #F2DEDE; + border: 1px solid #EED3D7; +} + +.parsley-errors-list { + margin: 2px 0 3px; + padding: 0; + list-style-type: none; + font-size: 0.9em; + line-height: 0.9em; + opacity: 0; + color: #B94A48; + + transition: all .3s ease-in; + -o-transition: all .3s ease-in; + -moz-transition: all .3s ease-in; + -webkit-transition: all .3s ease-in; +} + +.parsley-errors-list.filled { + opacity: 1; +} +/* END Parsley CSS */ \ No newline at end of file diff --git a/templates/base.html b/templates/base.html index 6a7487d4..15476cce 100644 --- a/templates/base.html +++ b/templates/base.html @@ -77,8 +77,9 @@ - - + + + {% if PLAUSIBLE_HOST and PLAUSIBLE_DOMAIN %} From ccb526faa14d798f778070bfb9b8ec650192be6f Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 6 Sep 2021 19:10:54 +0200 Subject: [PATCH 035/144] blur out other aliases when an alias is highlighted --- app/dashboard/templates/dashboard/index.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index 2cc67e1f..ae8f98e2 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -231,8 +231,12 @@ {% for alias_info in alias_infos %} {% set alias = alias_info.alias %} -
-
+
+
From b53da25a41eeb51eb208addb9059f410a4eeae98 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 6 Sep 2021 19:44:18 +0200 Subject: [PATCH 036/144] handle hotmail complaint --- app/config.py | 4 ++ app/email/status.py | 2 + app/email_utils.py | 13 ++++++ email_handler.py | 43 +++++++++++++++++++ .../transactional/hotmail-complaint.html | 33 ++++++++++++++ .../hotmail-complaint.txt.jinja2 | 17 ++++++++ 6 files changed, 112 insertions(+) create mode 100644 templates/emails/transactional/hotmail-complaint.html create mode 100644 templates/emails/transactional/hotmail-complaint.txt.jinja2 diff --git a/app/config.py b/app/config.py index c49109e8..1b3c869b 100644 --- a/app/config.py +++ b/app/config.py @@ -333,6 +333,8 @@ AlERT_WRONG_MX_RECORD_CUSTOM_DOMAIN = "custom_domain_mx_record_issue" # alert when a new alias is about to be created on a disabled directory ALERT_DIRECTORY_DISABLED_ALIAS_CREATION = "alert_directory_disabled_alias_creation" +ALERT_HOTMAIL_COMPLAINT = "alert_hotmail_complaint" + # <<<<< END ALERT EMAIL >>>> # Disable onboarding emails @@ -393,3 +395,5 @@ except Exception: HIBP_API_KEYS = sl_getenv("HIBP_API_KEYS", list) or [] NEWRELIC_CONFIG_PATH = os.environ.get("NEWRELIC_CONFIG_PATH") + +POSTMASTER = os.environ.get("POSTMASTER") \ No newline at end of file diff --git a/app/email/status.py b/app/email/status.py index 7811c937..28e95ae7 100644 --- a/app/email/status.py +++ b/app/email/status.py @@ -11,6 +11,8 @@ E206 = "250 SL E206 Out of office" # if mail_from is a IgnoreBounceSender, no need to send back a bounce report E207 = "250 SL E207 No bounce report" +E208 = "250 SL E208 Hotmail complaint handled" + # 4** errors # E401 = "421 SL E401 Retry later" E402 = "421 SL E402 Encryption failed - Retry later" diff --git a/app/email_utils.py b/app/email_utils.py index 6109dd3e..8f34b121 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -567,6 +567,19 @@ def get_orig_message_from_bounce(msg: Message) -> Message: return part +def get_orig_message_from_outlook_complaint(msg: Message) -> Message: + """parse the original email from Bounce""" + i = 0 + for part in msg.walk(): + i += 1 + + # 1st part is the container + # 2nd part is the empty body + # 3rd is original message + if i == 3: + return part + + def get_header_from_bounce(msg: Message, header: str) -> str: """using regex to get header value from bounce message get_orig_message_from_bounce is better. This should be the last option diff --git a/email_handler.py b/email_handler.py index fed44266..08ef4d84 100644 --- a/email_handler.py +++ b/email_handler.py @@ -76,6 +76,8 @@ from app.config import ( ENABLE_SPAM_ASSASSIN, BOUNCE_PREFIX_FOR_REPLY_PHASE, NEWRELIC_CONFIG_PATH, + POSTMASTER, + ALERT_HOTMAIL_COMPLAINT, ) from app.email import status from app.email.rate_limit import rate_limited @@ -112,6 +114,7 @@ from app.email_utils import ( sanitize_header, get_queue_id, should_ignore_bounce, + get_orig_message_from_outlook_complaint, ) from app.extensions import db from app.log import LOG, set_message_id @@ -1279,6 +1282,37 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): ) +def handle_hotmail_complaint(msg: Message): + """ + Handle hotmail complaint sent to postmaster + """ + orig_msg = get_orig_message_from_outlook_complaint(msg) + alias_address = orig_msg["To"] + alias = Alias.get_by(email=alias_address) + + if not alias: + LOG.d("No alias for %s", alias_address) + + user = alias.user + LOG.e("Handle hotmail complaint for %s %s", alias, user) + + send_email_with_rate_control( + user, + ALERT_HOTMAIL_COMPLAINT, + user.email, + f"Hotmail abuse report", + render( + "transactional/hotmail-complaint.txt.jinja2", + alias=alias, + ), + render( + "transactional/hotmail-complaint.html", + alias=alias, + ), + max_nb_alert=2, + ) + + def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog): """ Handle reply phase bounce @@ -1687,6 +1721,15 @@ def handle(envelope: Envelope) -> str: handle_transactional_bounce(envelope, rcpt_tos[0]) return status.E205 + if ( + len(rcpt_tos) == 1 + and mail_from == "staff@hotmail.com" + and rcpt_tos[0] == POSTMASTER + ): + LOG.w("Handle hotmail complaint") + handle_hotmail_complaint(msg) + return status.E208 + # Handle bounce if ( len(rcpt_tos) == 1 diff --git a/templates/emails/transactional/hotmail-complaint.html b/templates/emails/transactional/hotmail-complaint.html new file mode 100644 index 00000000..6c67c44e --- /dev/null +++ b/templates/emails/transactional/hotmail-complaint.html @@ -0,0 +1,33 @@ +{% extends "base.html" %} + +{% block content %} + {% call text() %} + This is SimpleLogin team.
+ Hotmail has informed us about an email sent to {{ alias.email }} that might have been marked as spam. + {% endcall %} + + {% call text() %} + Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and + is a violation of our terms and condition. + {% endcall %} + + {% call text() %} + If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. + {% endcall %} + + {% call text() %} + If SimpleLogin isn’t useful for you, please know that you can simply delete your account on the Settings page. + {% endcall %} + + {% call text() %} + Looking to hear back from you. + {% endcall %} + + {% call text() %} + Best,
+ SimpleLogin Team. + {% endcall %} + +{% endblock %} + + diff --git a/templates/emails/transactional/hotmail-complaint.txt.jinja2 b/templates/emails/transactional/hotmail-complaint.txt.jinja2 new file mode 100644 index 00000000..1e8d611e --- /dev/null +++ b/templates/emails/transactional/hotmail-complaint.txt.jinja2 @@ -0,0 +1,17 @@ +Hi, + +This is SimpleLogin team. + +Hotmail has informed us about an email sent to {{ alias.email }} that might have been marked as spam. + +Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and + is a violation of our terms and condition. + +If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. + +If SimpleLogin isn’t useful for you, please know that you can simply delete your account on the Settings page. + +Looking to hear back from you. + +Best, +SimpleLogin Team. \ No newline at end of file From 6dd6b740731377c106493276509b550497083c50 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 7 Sep 2021 11:13:28 +0200 Subject: [PATCH 037/144] black --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 1b3c869b..a7a7073f 100644 --- a/app/config.py +++ b/app/config.py @@ -396,4 +396,4 @@ HIBP_API_KEYS = sl_getenv("HIBP_API_KEYS", list) or [] NEWRELIC_CONFIG_PATH = os.environ.get("NEWRELIC_CONFIG_PATH") -POSTMASTER = os.environ.get("POSTMASTER") \ No newline at end of file +POSTMASTER = os.environ.get("POSTMASTER") From b2ac1b537d7e8c1f2a4996942f34b7a7d6e1d982 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 7 Sep 2021 15:10:37 +0200 Subject: [PATCH 038/144] add only pinned alias filter --- app/api/serializer.py | 6 ++++-- app/dashboard/templates/dashboard/index.html | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/api/serializer.py b/app/api/serializer.py index 91849a46..47fad9b2 100644 --- a/app/api/serializer.py +++ b/app/api/serializer.py @@ -245,6 +245,10 @@ def get_alias_infos_with_pagination_v3( q = q.filter(Alias.enabled) elif alias_filter == "disabled": q = q.filter(Alias.enabled.is_(False)) + elif alias_filter == "pinned": + q = q.filter(Alias.pinned) + elif alias_filter == "hibp": + q = q.filter(Alias.hibp_breaches.any()) q = q.order_by(Alias.pinned.desc()) @@ -256,8 +260,6 @@ def get_alias_infos_with_pagination_v3( q = q.order_by(Alias.email) elif sort == "z2a": q = q.order_by(Alias.email.desc()) - elif alias_filter == "hibp": - q = q.filter(Alias.hibp_breaches.any()) else: # default sorting q = q.order_by(latest_activity.desc()) diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index ae8f98e2..2035f550 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -177,6 +177,9 @@ + From 48838eb176702f1d3316e1b6cb79f7b9c517c9fc Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 7 Sep 2021 15:22:50 +0200 Subject: [PATCH 039/144] add highlighted alias in case it's not included in the result --- app/api/serializer.py | 194 ++++++++++++++++++++--------------- app/dashboard/views/index.py | 12 ++- 2 files changed, 123 insertions(+), 83 deletions(-) diff --git a/app/api/serializer.py b/app/api/serializer.py index 47fad9b2..f31dfee4 100644 --- a/app/api/serializer.py +++ b/app/api/serializer.py @@ -7,7 +7,15 @@ from sqlalchemy.orm import joinedload from app.config import PAGE_LIMIT from app.extensions import db -from app.models import Alias, Contact, EmailLog, Mailbox, AliasMailbox, CustomDomain +from app.models import ( + Alias, + Contact, + EmailLog, + Mailbox, + AliasMailbox, + CustomDomain, + User, +) @dataclass @@ -137,87 +145,7 @@ def get_alias_infos_with_pagination_v3( mailbox_id=None, directory_id=None, ) -> [AliasInfo]: - # subquery on alias annotated with nb_reply, nb_blocked, nb_forward, max_created_at, latest_email_log_created_at - alias_activity_subquery = ( - db.session.query( - Alias.id, - func.sum(case([(EmailLog.is_reply, 1)], else_=0)).label("nb_reply"), - func.sum( - case( - [(and_(EmailLog.is_reply.is_(False), EmailLog.blocked), 1)], - else_=0, - ) - ).label("nb_blocked"), - func.sum( - case( - [ - ( - and_( - EmailLog.is_reply.is_(False), - EmailLog.blocked.is_(False), - ), - 1, - ) - ], - else_=0, - ) - ).label("nb_forward"), - func.max(EmailLog.created_at).label("latest_email_log_created_at"), - ) - .join(EmailLog, Alias.id == EmailLog.alias_id, isouter=True) - .filter(Alias.user_id == user.id) - .group_by(Alias.id) - .subquery() - ) - - alias_contact_subquery = ( - db.session.query(Alias.id, func.max(Contact.id).label("max_contact_id")) - .join(Contact, Alias.id == Contact.alias_id, isouter=True) - .filter(Alias.user_id == user.id) - .group_by(Alias.id) - .subquery() - ) - - latest_activity = case( - [ - (Alias.created_at > EmailLog.created_at, Alias.created_at), - (Alias.created_at < EmailLog.created_at, EmailLog.created_at), - ], - else_=Alias.created_at, - ) - - q = ( - db.session.query( - Alias, - Contact, - EmailLog, - CustomDomain, - alias_activity_subquery.c.nb_reply, - alias_activity_subquery.c.nb_blocked, - alias_activity_subquery.c.nb_forward, - ) - .options(joinedload(Alias.hibp_breaches)) - .join(Contact, Alias.id == Contact.alias_id, isouter=True) - .join(CustomDomain, Alias.custom_domain_id == CustomDomain.id, isouter=True) - .join(EmailLog, Contact.id == EmailLog.contact_id, isouter=True) - .filter(Alias.id == alias_activity_subquery.c.id) - .filter(Alias.id == alias_contact_subquery.c.id) - .filter( - or_( - EmailLog.created_at - == alias_activity_subquery.c.latest_email_log_created_at, - and_( - # no email log yet for this alias - alias_activity_subquery.c.latest_email_log_created_at.is_(None), - # to make sure only 1 contact is returned in this case - or_( - Contact.id == alias_contact_subquery.c.max_contact_id, - alias_contact_subquery.c.max_contact_id.is_(None), - ), - ), - ) - ) - ) + q = construct_alias_query(user) if query: q = q.filter( @@ -262,6 +190,13 @@ def get_alias_infos_with_pagination_v3( q = q.order_by(Alias.email.desc()) else: # default sorting + latest_activity = case( + [ + (Alias.created_at > EmailLog.created_at, Alias.created_at), + (Alias.created_at < EmailLog.created_at, EmailLog.created_at), + ], + else_=Alias.created_at, + ) q = q.order_by(latest_activity.desc()) q = list(q.limit(PAGE_LIMIT).offset(page_id * PAGE_LIMIT)) @@ -374,3 +309,98 @@ def get_alias_contacts(alias, page_id: int) -> [dict]: res.append(serialize_contact(fe)) return res + + +def get_alias_info_v3(user: User, alias_id: int) -> AliasInfo: + # use the same query construction in get_alias_infos_with_pagination_v3 + q = construct_alias_query(user) + q = q.filter(Alias.id == alias_id) + + for alias, contact, email_log, custom_domain, nb_reply, nb_blocked, nb_forward in q: + return AliasInfo( + alias=alias, + mailbox=alias.mailbox, + mailboxes=alias.mailboxes, + nb_forward=nb_forward, + nb_blocked=nb_blocked, + nb_reply=nb_reply, + latest_email_log=email_log, + latest_contact=contact, + custom_domain=custom_domain, + ) + + +def construct_alias_query(user: User): + # subquery on alias annotated with nb_reply, nb_blocked, nb_forward, max_created_at, latest_email_log_created_at + alias_activity_subquery = ( + db.session.query( + Alias.id, + func.sum(case([(EmailLog.is_reply, 1)], else_=0)).label("nb_reply"), + func.sum( + case( + [(and_(EmailLog.is_reply.is_(False), EmailLog.blocked), 1)], + else_=0, + ) + ).label("nb_blocked"), + func.sum( + case( + [ + ( + and_( + EmailLog.is_reply.is_(False), + EmailLog.blocked.is_(False), + ), + 1, + ) + ], + else_=0, + ) + ).label("nb_forward"), + func.max(EmailLog.created_at).label("latest_email_log_created_at"), + ) + .join(EmailLog, Alias.id == EmailLog.alias_id, isouter=True) + .filter(Alias.user_id == user.id) + .group_by(Alias.id) + .subquery() + ) + + alias_contact_subquery = ( + db.session.query(Alias.id, func.max(Contact.id).label("max_contact_id")) + .join(Contact, Alias.id == Contact.alias_id, isouter=True) + .filter(Alias.user_id == user.id) + .group_by(Alias.id) + .subquery() + ) + + return ( + db.session.query( + Alias, + Contact, + EmailLog, + CustomDomain, + alias_activity_subquery.c.nb_reply, + alias_activity_subquery.c.nb_blocked, + alias_activity_subquery.c.nb_forward, + ) + .options(joinedload(Alias.hibp_breaches)) + .join(Contact, Alias.id == Contact.alias_id, isouter=True) + .join(CustomDomain, Alias.custom_domain_id == CustomDomain.id, isouter=True) + .join(EmailLog, Contact.id == EmailLog.contact_id, isouter=True) + .filter(Alias.id == alias_activity_subquery.c.id) + .filter(Alias.id == alias_contact_subquery.c.id) + .filter( + or_( + EmailLog.created_at + == alias_activity_subquery.c.latest_email_log_created_at, + and_( + # no email log yet for this alias + alias_activity_subquery.c.latest_email_log_created_at.is_(None), + # to make sure only 1 contact is returned in this case + or_( + Contact.id == alias_contact_subquery.c.max_contact_id, + alias_contact_subquery.c.max_contact_id.is_(None), + ), + ), + ) + ) + ) diff --git a/app/dashboard/views/index.py b/app/dashboard/views/index.py index 51690075..5cc61c27 100644 --- a/app/dashboard/views/index.py +++ b/app/dashboard/views/index.py @@ -4,7 +4,7 @@ from flask import render_template, request, redirect, url_for, flash from flask_login import login_required, current_user from app import alias_utils -from app.api.serializer import get_alias_infos_with_pagination_v3 +from app.api.serializer import get_alias_infos_with_pagination_v3, get_alias_info_v3 from app.config import PAGE_LIMIT, ALIAS_LIMIT from app.dashboard.base import dashboard_bp from app.extensions import db, limiter @@ -163,6 +163,16 @@ def index(): ) last_page = len(alias_infos) < PAGE_LIMIT + # add highlighted alias in case it's not included + if highlight_alias_id and highlight_alias_id not in [ + alias_info.alias.id for alias_info in alias_infos + ]: + highlight_alias_info = get_alias_info_v3( + current_user, alias_id=highlight_alias_id + ) + if highlight_alias_info: + alias_infos.insert(0, highlight_alias_info) + return render_template( "dashboard/index.html", alias_infos=alias_infos, From d9f44437da02fd48003f69db1911565fd30fdfbd Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 7 Sep 2021 15:35:55 +0200 Subject: [PATCH 040/144] add Coupon.is_giveaway column --- app/models.py | 4 +++ .../versions/2021_090715_916a5257d18c_.py | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 migrations/versions/2021_090715_916a5257d18c_.py diff --git a/app/models.py b/app/models.py index ce97d383..85ac66c0 100644 --- a/app/models.py +++ b/app/models.py @@ -1917,6 +1917,10 @@ class Coupon(db.Model, ModelMixin): db.ForeignKey(User.id, ondelete="cascade"), nullable=True ) + is_giveaway = db.Column( + db.Boolean, default=False, nullable=False, server_default="0" + ) + class Directory(db.Model, ModelMixin): user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False) diff --git a/migrations/versions/2021_090715_916a5257d18c_.py b/migrations/versions/2021_090715_916a5257d18c_.py new file mode 100644 index 00000000..1d71eaa1 --- /dev/null +++ b/migrations/versions/2021_090715_916a5257d18c_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: 916a5257d18c +Revises: 424808e1fe49 +Create Date: 2021-09-07 15:35:35.430202 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '916a5257d18c' +down_revision = '424808e1fe49' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('coupon', sa.Column('is_giveaway', sa.Boolean(), server_default='0', nullable=False)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('coupon', 'is_giveaway') + # ### end Alembic commands ### From f3cff1f1bfd6614d201eef5204d955cae20a850b Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 7 Sep 2021 15:36:19 +0200 Subject: [PATCH 041/144] create manual subscription using the coupon.is_giveaway info --- app/dashboard/views/coupon.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/dashboard/views/coupon.py b/app/dashboard/views/coupon.py index dacbd2da..7b3228fd 100644 --- a/app/dashboard/views/coupon.py +++ b/app/dashboard/views/coupon.py @@ -78,7 +78,7 @@ def coupon_route(): user_id=current_user.id, end_at=arrow.now().shift(years=coupon.nb_year, days=1), comment="using coupon code", - is_giveaway=False, + is_giveaway=coupon.is_giveaway, commit=True, ) flash( @@ -87,9 +87,13 @@ def coupon_route(): ) # notify admin + if coupon.is_giveaway: + subject = f"User {current_user} applies a (free) coupon" + else: + subject = f"User {current_user} applies a (paid) coupon" send_email( ADMIN_EMAIL, - subject=f"User {current_user} applies the coupon", + subject=subject, plaintext="", html="", ) From 41f10373d18fb7dcd105e417484527b1fbd21a25 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 10:51:47 +0200 Subject: [PATCH 042/144] add plan info --- app/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models.py b/app/models.py index 85ac66c0..170e8770 100644 --- a/app/models.py +++ b/app/models.py @@ -505,9 +505,9 @@ class User(db.Model, ModelMixin, UserMixin, PasswordOracle): sub: Subscription = self.get_subscription() if sub: if sub.cancelled: - return f"Cancelled Paddle Subscription {sub.subscription_id}" + return f"Cancelled Paddle Subscription {sub.subscription_id} {sub.plan_name()}" else: - return f"Active Paddle Subscription {sub.subscription_id}" + return f"Active Paddle Subscription {sub.subscription_id} {sub.plan_name()}" apple_sub: AppleSubscription = AppleSubscription.get_by(user_id=self.id) if apple_sub and apple_sub.is_valid(): From 201eb3b9b9575894615a76c5c244ae73c276ebdb Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 11:02:35 +0200 Subject: [PATCH 043/144] fix handle_hotmail_complaint: handle email format with name --- email_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 08ef4d84..177b1458 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1287,11 +1287,13 @@ def handle_hotmail_complaint(msg: Message): Handle hotmail complaint sent to postmaster """ orig_msg = get_orig_message_from_outlook_complaint(msg) - alias_address = orig_msg["To"] + to_header = orig_msg["To"] + _, alias_address = parseaddr_unicode(to_header) alias = Alias.get_by(email=alias_address) if not alias: LOG.d("No alias for %s", alias_address) + return user = alias.user LOG.e("Handle hotmail complaint for %s %s", alias, user) From a6e8684afb09a038293817218178b88e72fb13d2 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 11:25:53 +0200 Subject: [PATCH 044/144] add more debug info --- email_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/email_handler.py b/email_handler.py index 177b1458..530a54a1 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1702,9 +1702,11 @@ def handle(envelope: Envelope) -> str: contact = Contact.get_by(reply_email=mail_from) if contact: LOG.e( - "email can't be sent from a reverse-alias alias:%s, contact email:%s", - contact.alias, + "email can't be sent from a reverse-alias:%s, contact email:%s, %s, %s", + contact.reply_email, contact.website_email, + contact.alias, + contact.user, ) return status.E203 From 5ac78f2694762be8cdec4b6498b1edbc574059b7 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 11:29:55 +0200 Subject: [PATCH 045/144] reformat --- app/alias_utils.py | 16 +++++------ app/api/views/apple.py | 18 ++++++------- app/api/views/auth.py | 4 +-- app/api/views/new_custom_alias.py | 8 +++--- app/api/views/setting.py | 2 +- app/auth/views/activate.py | 4 +-- app/auth/views/facebook.py | 2 +- app/auth/views/fido.py | 2 +- app/auth/views/github.py | 2 +- app/auth/views/google.py | 2 +- app/auth/views/login.py | 2 +- app/auth/views/login_utils.py | 6 ++--- app/auth/views/recovery.py | 4 +-- app/auth/views/register.py | 4 +-- app/dashboard/views/batch_import.py | 2 +- app/dashboard/views/billing.py | 6 ++--- app/dashboard/views/custom_alias.py | 16 +++++------ app/dashboard/views/enter_sudo.py | 4 +-- app/dashboard/views/fido_setup.py | 2 +- app/dashboard/views/index.py | 2 +- app/dashboard/views/refused_email.py | 2 +- app/dashboard/views/setting.py | 6 ++--- app/email/spam.py | 2 +- app/email_utils.py | 40 +++++++++++++--------------- app/import_utils.py | 6 ++--- app/jose_utils.py | 2 +- app/models.py | 22 +++++++-------- app/oauth/views/authorize.py | 18 ++++++------- app/oauth/views/token.py | 4 +-- app/paddle_utils.py | 6 ++--- app/pgp_utils.py | 4 +-- app/spamassassin_utils.py | 2 +- cron.py | 38 ++++++++++++-------------- email_handler.py | 22 +++++++-------- init_app.py | 12 +++------ job_runner.py | 4 +-- monitoring.py | 2 +- server.py | 28 +++++++++---------- shell.py | 10 +++---- 39 files changed, 157 insertions(+), 181 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index 7e0f4629..a5c4cd81 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -31,9 +31,7 @@ from app.models import ( def try_auto_create(address: str) -> Optional[Alias]: """Try to auto-create the alias using directory or catch-all domain""" if address.startswith(f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+"): - LOG.exception( - "alias %s can't start with %s", address, BOUNCE_PREFIX_FOR_REPLY_PHASE - ) + LOG.e("alias %s can't start with %s", address, BOUNCE_PREFIX_FOR_REPLY_PHASE) return None alias = try_auto_create_catch_all_domain(address) @@ -101,7 +99,7 @@ def try_auto_create_directory(address: str) -> Optional[Alias]: db.session.commit() return alias except AliasInTrashError: - LOG.warning( + LOG.w( "Alias %s was deleted before, cannot auto-create using directory %s, user %s", address, directory_name, @@ -109,7 +107,7 @@ def try_auto_create_directory(address: str) -> Optional[Alias]: ) return None except IntegrityError: - LOG.warning("Alias %s already exists", address) + LOG.w("Alias %s already exists", address) db.session.rollback() alias = Alias.get_by(email=address) return alias @@ -156,7 +154,7 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: db.session.commit() return alias except AliasInTrashError: - LOG.warning( + LOG.w( "Alias %s was deleted before, cannot auto-create using domain catch-all %s, user %s", address, custom_domain, @@ -164,12 +162,12 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: ) return None except IntegrityError: - LOG.warning("Alias %s already exists", address) + LOG.w("Alias %s already exists", address) db.session.rollback() alias = Alias.get_by(email=address) return alias except DataError: - LOG.warning("Cannot create alias %s", address) + LOG.w("Cannot create alias %s", address) db.session.rollback() return None @@ -184,7 +182,7 @@ def delete_alias(alias: Alias, user: User): if not DomainDeletedAlias.get_by( email=alias.email, domain_id=alias.custom_domain_id ): - LOG.debug("add %s to domain %s trash", alias, alias.custom_domain_id) + LOG.d("add %s to domain %s trash", alias, alias.custom_domain_id) db.session.add( DomainDeletedAlias( user_id=user.id, email=alias.email, domain_id=alias.custom_domain_id diff --git a/app/api/views/apple.py b/app/api/views/apple.py index bcc8acfc..563c9d0b 100644 --- a/app/api/views/apple.py +++ b/app/api/views/apple.py @@ -36,7 +36,7 @@ def apple_process_payment(): 200 of the payment is successful, i.e. user is upgraded to premium """ - LOG.debug("request for /apple/process_payment") + LOG.d("request for /apple/process_payment") user = g.user data = request.get_json() receipt_data = data.get("receipt_data") @@ -229,7 +229,7 @@ def apple_update_notification(): # "auto_renew_product_id": "io.simplelogin.ios_app.subscription.premium.yearly", # "notification_type": "DID_CHANGE_RENEWAL_STATUS", # } - LOG.debug("request for /api/apple/update_notification") + LOG.d("request for /api/apple/update_notification") data = request.get_json() if not ( data @@ -282,7 +282,7 @@ def apple_update_notification(): db.session.commit() return jsonify(ok=True), 200 else: - LOG.warning( + LOG.w( "No existing AppleSub for original_transaction_id %s", original_transaction_id, ) @@ -305,16 +305,16 @@ def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]: _PROD_URL, json={"receipt-data": receipt_data, "password": password} ) except RequestException: - LOG.warning("cannot call Apple server %s", _PROD_URL) + LOG.w("cannot call Apple server %s", _PROD_URL) return None if r.status_code >= 500: - LOG.warning("Apple server error, response:%s %s", r, r.content) + LOG.w("Apple server error, response:%s %s", r, r.content) return None if r.json() == {"status": 21007}: # try sandbox_url - LOG.warning("Use the sandbox url instead") + LOG.w("Use the sandbox url instead") r = requests.post( _SANDBOX_URL, json={"receipt-data": receipt_data, "password": password}, @@ -472,7 +472,7 @@ def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]: # } if data["status"] != 0: - LOG.warning( + LOG.w( "verifyReceipt status !=0, probably invalid receipt. User %s", user, ) @@ -499,7 +499,7 @@ def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]: # } transactions = data["receipt"]["in_app"] if not transactions: - LOG.warning("Empty transactions in data %s", data) + LOG.w("Empty transactions in data %s", data) return None latest_transaction = max(transactions, key=lambda t: int(t["expires_date_ms"])) @@ -527,7 +527,7 @@ def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]: else: # the same original_transaction_id has been used on another account if AppleSubscription.get_by(original_transaction_id=original_transaction_id): - LOG.exception("Same Apple Sub has been used before, current user %s", user) + LOG.e("Same Apple Sub has been used before, current user %s", user) return None LOG.d( diff --git a/app/api/views/auth.py b/app/api/views/auth.py index fd6bc588..4b7a072f 100644 --- a/app/api/views/auth.py +++ b/app/api/views/auth.py @@ -96,7 +96,7 @@ def auth_register(): if not password or len(password) < 8: return jsonify(error="password too short"), 400 - LOG.debug("create user %s", email) + LOG.d("create user %s", email) user = User.create(email=email, name="", password=password) db.session.flush() @@ -166,7 +166,7 @@ def auth_activate(): return jsonify(error="Wrong email or code"), 400 - LOG.debug("activate user %s", user) + LOG.d("activate user %s", user) user.activated = True AccountActivation.delete(account_activation.id) db.session.commit() diff --git a/app/api/views/new_custom_alias.py b/app/api/views/new_custom_alias.py index 762456a1..5148da72 100644 --- a/app/api/views/new_custom_alias.py +++ b/app/api/views/new_custom_alias.py @@ -69,10 +69,10 @@ def new_custom_alias_v2(): try: alias_suffix = signer.unsign(signed_suffix, max_age=600).decode() except SignatureExpired: - LOG.warning("Alias creation time expired for %s", user) + LOG.w("Alias creation time expired for %s", user) return jsonify(error="Alias creation time is expired, please retry"), 412 except Exception: - LOG.warning("Alias suffix is tampered, user %s", user) + LOG.w("Alias suffix is tampered, user %s", user) return jsonify(error="Tampered suffix"), 400 if not verify_prefix_suffix(user, alias_prefix, alias_suffix): @@ -184,10 +184,10 @@ def new_custom_alias_v3(): try: alias_suffix = signer.unsign(signed_suffix, max_age=600).decode() except SignatureExpired: - LOG.warning("Alias creation time expired for %s", user) + LOG.w("Alias creation time expired for %s", user) return jsonify(error="Alias creation time is expired, please retry"), 412 except Exception: - LOG.warning("Alias suffix is tampered, user %s", user) + LOG.w("Alias suffix is tampered, user %s", user) return jsonify(error="Tampered suffix"), 400 if not verify_prefix_suffix(user, alias_prefix, alias_suffix): diff --git a/app/api/views/setting.py b/app/api/views/setting.py index 73a89598..a209265e 100644 --- a/app/api/views/setting.py +++ b/app/api/views/setting.py @@ -87,7 +87,7 @@ def update_setting(): # sanity check if custom_domain.user_id != user.id or not custom_domain.verified: - LOG.exception("%s cannot use domain %s", user, default_domain) + LOG.e("%s cannot use domain %s", user, default_domain) return jsonify(error="invalid domain"), 400 else: user.default_alias_custom_domain_id = custom_domain.id diff --git a/app/auth/views/activate.py b/app/auth/views/activate.py index 9324905d..e9943f99 100644 --- a/app/auth/views/activate.py +++ b/app/auth/views/activate.py @@ -57,8 +57,8 @@ def activate(): # The activation link contains the original page, for ex authorize page if "next" in request.args: next_url = request.args.get("next") - LOG.debug("redirect user to %s", next_url) + LOG.d("redirect user to %s", next_url) return redirect(next_url) else: - LOG.debug("redirect user to dashboard") + LOG.d("redirect user to dashboard") return redirect(url_for("dashboard.index")) diff --git a/app/auth/views/facebook.py b/app/auth/views/facebook.py index f3c7b0e4..66d2d695 100644 --- a/app/auth/views/facebook.py +++ b/app/auth/views/facebook.py @@ -115,7 +115,7 @@ def facebook_callback(): # The activation link contains the original page, for ex authorize page if "facebook_next_url" in session: next_url = session["facebook_next_url"] - LOG.debug("redirect user to %s", next_url) + LOG.d("redirect user to %s", next_url) # reset the next_url to avoid user getting redirected at each login :) session.pop("facebook_next_url", None) diff --git a/app/auth/views/fido.py b/app/auth/views/fido.py index c629df49..aa1ed063 100644 --- a/app/auth/views/fido.py +++ b/app/auth/views/fido.py @@ -95,7 +95,7 @@ def fido(): ) new_sign_count = webauthn_assertion_response.verify() except Exception as e: - LOG.warning(f"An error occurred in WebAuthn verification process: {e}") + LOG.w(f"An error occurred in WebAuthn verification process: {e}") flash("Key verification failed.", "warning") # Trigger rate limiter g.deduct_limit = True diff --git a/app/auth/views/github.py b/app/auth/views/github.py index d130705e..99a93dde 100644 --- a/app/auth/views/github.py +++ b/app/auth/views/github.py @@ -75,7 +75,7 @@ def github_callback(): break if not email: - LOG.exception(f"cannot get email for github user {github_user_data} {emails}") + LOG.e(f"cannot get email for github user {github_user_data} {emails}") flash( "Cannot get a valid email from Github, please another way to login/sign up", "error", diff --git a/app/auth/views/google.py b/app/auth/views/google.py index 66336c93..08dddc3d 100644 --- a/app/auth/views/google.py +++ b/app/auth/views/google.py @@ -101,7 +101,7 @@ def google_callback(): # The activation link contains the original page, for ex authorize page if "google_next_url" in session: next_url = session["google_next_url"] - LOG.debug("redirect user to %s", next_url) + LOG.d("redirect user to %s", next_url) # reset the next_url to avoid user getting redirected at each login :) session.pop("google_next_url", None) diff --git a/app/auth/views/login.py b/app/auth/views/login.py index 4d7ae548..f9c87f7d 100644 --- a/app/auth/views/login.py +++ b/app/auth/views/login.py @@ -25,7 +25,7 @@ def login(): if current_user.is_authenticated: if next_url: - LOG.debug("user is already authenticated, redirect to %s", next_url) + LOG.d("user is already authenticated, redirect to %s", next_url) return redirect(next_url) else: LOG.d("user is already authenticated, redirect to dashboard") diff --git a/app/auth/views/login_utils.py b/app/auth/views/login_utils.py index 9bd2b7ad..c4426d06 100644 --- a/app/auth/views/login_utils.py +++ b/app/auth/views/login_utils.py @@ -29,15 +29,15 @@ def after_login(user, next_url): else: return redirect(url_for("auth.mfa")) else: - LOG.debug("log user %s in", user) + LOG.d("log user %s in", user) login_user(user) # User comes to login page from another page if next_url: - LOG.debug("redirect user to %s", next_url) + LOG.d("redirect user to %s", next_url) return redirect(next_url) else: - LOG.debug("redirect user to dashboard") + LOG.d("redirect user to dashboard") return redirect(url_for("dashboard.index")) diff --git a/app/auth/views/recovery.py b/app/auth/views/recovery.py index aa788d47..fac50932 100644 --- a/app/auth/views/recovery.py +++ b/app/auth/views/recovery.py @@ -58,10 +58,10 @@ def recovery_route(): # User comes to login page from another page if next_url: - LOG.debug("redirect user to %s", next_url) + LOG.d("redirect user to %s", next_url) return redirect(next_url) else: - LOG.debug("redirect user to dashboard") + LOG.d("redirect user to dashboard") return redirect(url_for("dashboard.index")) else: # Trigger rate limiter diff --git a/app/auth/views/register.py b/app/auth/views/register.py index a594d69d..30714cdf 100644 --- a/app/auth/views/register.py +++ b/app/auth/views/register.py @@ -53,7 +53,7 @@ def register(): # 'challenge_ts': '2020-07-23T10:03:25', # 'hostname': '127.0.0.1'} if not hcaptcha_res["success"]: - LOG.warning( + LOG.w( "User put wrong captcha %s %s", form.email.data, hcaptcha_res, @@ -74,7 +74,7 @@ def register(): if personal_email_already_used(email): flash(f"Email {email} already used", "error") else: - LOG.debug("create user %s", email) + LOG.d("create user %s", email) user = User.create( email=email, name="", diff --git a/app/dashboard/views/batch_import.py b/app/dashboard/views/batch_import.py index 1e190dfe..343a05f4 100644 --- a/app/dashboard/views/batch_import.py +++ b/app/dashboard/views/batch_import.py @@ -31,7 +31,7 @@ def batch_import_route(): bi = BatchImport.create(user_id=current_user.id, file_id=file.id) db.session.flush() - LOG.debug("Add a batch import job %s for %s", bi, current_user) + LOG.d("Add a batch import job %s for %s", bi, current_user) # Schedule batch import job Job.create( diff --git a/app/dashboard/views/billing.py b/app/dashboard/views/billing.py index 4f3b6b01..d4cb9297 100644 --- a/app/dashboard/views/billing.py +++ b/app/dashboard/views/billing.py @@ -21,7 +21,7 @@ def billing(): if request.method == "POST": if request.form.get("form-name") == "cancel": - LOG.warning(f"User {current_user} cancels their subscription") + LOG.w(f"User {current_user} cancels their subscription") success = cancel_subscription(sub.subscription_id) if success: @@ -37,7 +37,7 @@ def billing(): return redirect(url_for("dashboard.billing")) elif request.form.get("form-name") == "change-monthly": - LOG.debug(f"User {current_user} changes to monthly plan") + LOG.d(f"User {current_user} changes to monthly plan") success, msg = change_plan( current_user, sub.subscription_id, PADDLE_MONTHLY_PRODUCT_ID ) @@ -58,7 +58,7 @@ def billing(): return redirect(url_for("dashboard.billing")) elif request.form.get("form-name") == "change-yearly": - LOG.debug(f"User {current_user} changes to yearly plan") + LOG.d(f"User {current_user} changes to yearly plan") success, msg = change_plan( current_user, sub.subscription_id, PADDLE_YEARLY_PRODUCT_ID ) diff --git a/app/dashboard/views/custom_alias.py b/app/dashboard/views/custom_alias.py index d5c08cc0..a6eb5066 100644 --- a/app/dashboard/views/custom_alias.py +++ b/app/dashboard/views/custom_alias.py @@ -249,11 +249,11 @@ def custom_alias(): signed_alias_suffix_decoded ) except SignatureExpired: - LOG.warning("Alias creation time expired for %s", current_user) + LOG.w("Alias creation time expired for %s", current_user) flash("Alias creation time is expired, please retry", "warning") return redirect(url_for("dashboard.custom_alias")) except Exception: - LOG.warning("Alias suffix is tampered, user %s", current_user) + LOG.w("Alias suffix is tampered, user %s", current_user) flash("Unknown error, refresh the page", "error") return redirect(url_for("dashboard.custom_alias")) @@ -281,7 +281,7 @@ def custom_alias(): ) else: # should never happen as user can only choose their domains - LOG.exception( + LOG.e( "Deleted Alias %s does not belong to user %s", domain_deleted_alias, ) @@ -309,7 +309,7 @@ def custom_alias(): ) db.session.flush() except IntegrityError: - LOG.warning("Alias %s already exists", full_alias) + LOG.w("Alias %s already exists", full_alias) db.session.rollback() flash("Unknown error, please retry", "error") return redirect(url_for("dashboard.custom_alias")) @@ -351,7 +351,7 @@ def verify_prefix_suffix(user: User, alias_prefix, alias_suffix) -> bool: # alias_domain must be either one of user custom domains or built-in domains if alias_domain not in user.available_alias_domains(): - LOG.exception("wrong alias suffix %s, user %s", alias_suffix, user) + LOG.e("wrong alias suffix %s, user %s", alias_suffix, user) return False # SimpleLogin domain case: @@ -365,17 +365,17 @@ def verify_prefix_suffix(user: User, alias_prefix, alias_suffix) -> bool: ): if not alias_domain_prefix.startswith("."): - LOG.exception("User %s submits a wrong alias suffix %s", user, alias_suffix) + LOG.e("User %s submits a wrong alias suffix %s", user, alias_suffix) return False else: if alias_domain not in user_custom_domains: if not DISABLE_ALIAS_SUFFIX: - LOG.exception("wrong alias suffix %s, user %s", alias_suffix, user) + LOG.e("wrong alias suffix %s, user %s", alias_suffix, user) return False if alias_domain not in user.available_sl_domains(): - LOG.exception("wrong alias suffix %s, user %s", alias_suffix, user) + LOG.e("wrong alias suffix %s, user %s", alias_suffix, user) return False return True diff --git a/app/dashboard/views/enter_sudo.py b/app/dashboard/views/enter_sudo.py index e5f5a33c..6d388bdf 100644 --- a/app/dashboard/views/enter_sudo.py +++ b/app/dashboard/views/enter_sudo.py @@ -30,10 +30,10 @@ def enter_sudo(): # User comes to sudo page from another page next_url = request.args.get("next") if next_url: - LOG.debug("redirect user to %s", next_url) + LOG.d("redirect user to %s", next_url) return redirect(next_url) else: - LOG.debug("redirect user to dashboard") + LOG.d("redirect user to dashboard") return redirect(url_for("dashboard.index")) else: flash("Incorrect password", "warning") diff --git a/app/dashboard/views/fido_setup.py b/app/dashboard/views/fido_setup.py index c7efe46b..282336f8 100644 --- a/app/dashboard/views/fido_setup.py +++ b/app/dashboard/views/fido_setup.py @@ -55,7 +55,7 @@ def fido_setup(): try: fido_credential = fido_reg_response.verify() except Exception as e: - LOG.warning(f"An error occurred in WebAuthn registration process: {e}") + LOG.w(f"An error occurred in WebAuthn registration process: {e}") flash("Key registration failed.", "warning") return redirect(url_for("dashboard.index")) diff --git a/app/dashboard/views/index.py b/app/dashboard/views/index.py index 5cc61c27..d119ef2d 100644 --- a/app/dashboard/views/index.py +++ b/app/dashboard/views/index.py @@ -69,7 +69,7 @@ def index(): try: highlight_alias_id = int(request.args.get("highlight_alias_id")) except ValueError: - LOG.warning( + LOG.w( "highlight_alias_id must be a number, received %s", request.args.get("highlight_alias_id"), ) diff --git a/app/dashboard/views/refused_email.py b/app/dashboard/views/refused_email.py index c52a06a5..f3a5cab6 100644 --- a/app/dashboard/views/refused_email.py +++ b/app/dashboard/views/refused_email.py @@ -15,7 +15,7 @@ def refused_email_route(): try: highlight_id = int(highlight_id) except ValueError: - LOG.warning("Cannot parse highlight_id %s", highlight_id) + LOG.w("Cannot parse highlight_id %s", highlight_id) highlight_id = None email_logs: [EmailLog] = ( diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index 8eba0929..11b2a9cd 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -105,7 +105,7 @@ def setting(): other_email_change: EmailChange = EmailChange.get_by( new_email=new_email ) - LOG.warning( + LOG.w( "Another user has a pending %s with the same email address. Current user:%s", other_email_change, current_user, @@ -193,7 +193,7 @@ def setting(): return redirect(url_for("dashboard.setting")) # Schedule delete account job - LOG.warning("schedule delete account job for %s", current_user) + LOG.w("schedule delete account job for %s", current_user) Job.create( name=JOB_DELETE_ACCOUNT, payload={"user_id": current_user.id}, @@ -236,7 +236,7 @@ def setting(): custom_domain.user_id != current_user.id or not custom_domain.verified ): - LOG.exception( + LOG.e( "%s cannot use domain %s", current_user, default_domain ) else: diff --git a/app/email/spam.py b/app/email/spam.py index a83b2e90..c8a481be 100644 --- a/app/email/spam.py +++ b/app/email/spam.py @@ -59,5 +59,5 @@ def get_spam_score( return get_spam_score(message, email_log, can_retry=False) else: # return a negative score so the message is always considered as ham - LOG.exception("SpamAssassin exception, ignore spam check") + LOG.e("SpamAssassin exception, ignore spam check") return -999, None diff --git a/app/email_utils.py b/app/email_utils.py index 8f34b121..4eefbb6d 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -323,7 +323,7 @@ def send_email_with_rate_control( ) if nb_alert >= max_nb_alert: - LOG.warning( + LOG.w( "%s emails were sent to %s in the last %s days, alert type %s", nb_alert, to_email, @@ -359,7 +359,7 @@ def send_email_at_most_times( ).count() if nb_alert >= max_times: - LOG.warning( + LOG.w( "%s emails were sent to %s alert type %s", nb_alert, to_email, @@ -645,7 +645,7 @@ def get_spam_from_header(spam_status_header, max_score=None) -> (bool, str): ) score = float(score_section[len("score=") :]) if score >= max_score: - LOG.warning("Spam score %s exceeds %s", score, max_score) + LOG.w("Spam score %s exceeds %s", score, max_score) return True, spam_status_header return spamassassin_answer.lower() == "yes", spam_status_header @@ -660,13 +660,13 @@ def get_header_unicode(header: str) -> str: try: return decoded_string.decode(charset) except UnicodeDecodeError: - LOG.warning("Cannot decode header %s", header) + LOG.w("Cannot decode header %s", header) except LookupError: # charset is unknown - LOG.warning("Cannot decode %s with %s, use utf-8", decoded_string, charset) + LOG.w("Cannot decode %s with %s, use utf-8", decoded_string, charset) try: return decoded_string.decode("utf-8") except UnicodeDecodeError: - LOG.warning("Cannot UTF-8 decode %s", decoded_string) + LOG.w("Cannot UTF-8 decode %s", decoded_string) return decoded_string.decode("utf-8", errors="replace") return header @@ -687,16 +687,16 @@ def parseaddr_unicode(addr) -> (str, str): try: decoded_string, charset = decode_header(name)[0] except HeaderParseError: # fail in case - LOG.warning("Can't decode name %s", name) + LOG.w("Can't decode name %s", name) else: if charset is not None: try: name = decoded_string.decode(charset) except UnicodeDecodeError: - LOG.warning("Cannot decode addr name %s", name) + LOG.w("Cannot decode addr name %s", name) name = "" except LookupError: # charset is unknown - LOG.warning( + LOG.w( "Cannot decode %s with %s, use utf-8", decoded_string, charset ) name = decoded_string.decode("utf-8") @@ -713,11 +713,11 @@ def copy(msg: Message) -> Message: try: return deepcopy(msg) except Exception: - LOG.warning("deepcopy fails, try string parsing") + LOG.w("deepcopy fails, try string parsing") try: return email.message_from_string(msg.as_string()) except (UnicodeEncodeError, KeyError, LookupError): - LOG.warning("as_string() fails, try bytes parsing") + LOG.w("as_string() fails, try bytes parsing") return email.message_from_bytes(to_bytes(msg)) @@ -726,17 +726,15 @@ def to_bytes(msg: Message): try: return msg.as_bytes() except UnicodeEncodeError: - LOG.warning("as_bytes fails with default policy, try SMTP policy") + LOG.w("as_bytes fails with default policy, try SMTP policy") try: return msg.as_bytes(policy=email.policy.SMTP) except UnicodeEncodeError: - LOG.warning("as_bytes fails with SMTP policy, try SMTPUTF8 policy") + LOG.w("as_bytes fails with SMTP policy, try SMTPUTF8 policy") try: return msg.as_bytes(policy=email.policy.SMTPUTF8) except UnicodeEncodeError: - LOG.warning( - "as_bytes fails with SMTPUTF8 policy, try converting to string" - ) + LOG.w("as_bytes fails with SMTPUTF8 policy, try converting to string") msg_string = msg.as_string() try: return msg_string.encode() @@ -790,7 +788,7 @@ def get_encoding(msg: Message) -> EmailEncoding: if cte in ("amazonses.com",): return EmailEncoding.NO - LOG.exception("Unknown encoding %s", cte) + LOG.e("Unknown encoding %s", cte) return EmailEncoding.NO @@ -915,7 +913,7 @@ def replace(msg: Message, old, new) -> Message: clone_msg.set_payload(new_parts) return clone_msg - LOG.exception("Cannot replace text for %s", msg.get_content_type()) + LOG.e("Cannot replace text for %s", msg.get_content_type()) return msg @@ -991,7 +989,7 @@ def should_disable(alias: Alias) -> bool: """Disable an alias if it has too many bounces recently""" # Bypass the bounce rule if alias.cannot_be_disabled: - LOG.warning("%s cannot be disabled", alias) + LOG.w("%s cannot be disabled", alias) return False yesterday = arrow.now().shift(days=-1) @@ -1025,7 +1023,7 @@ def should_disable(alias: Alias) -> bool: .count() ) if nb_bounced_7d_1d > 1: - LOG.debug( + LOG.d( "more than 5 bounces in the last 24h and more than 1 bounces in the last 7 days, " "disable alias %s", alias, @@ -1105,7 +1103,7 @@ def spf_pass( try: r = spf.check2(i=ip, s=envelope.mail_from, h=None) except Exception: - LOG.exception("SPF error, mailbox %s, ip %s", mailbox.email, ip) + LOG.e("SPF error, mailbox %s, ip %s", mailbox.email, ip) else: # TODO: Handle temperr case (e.g. dns timeout) # only an absolute pass, or no SPF policy at all is 'valid' diff --git a/app/import_utils.py b/app/import_utils.py index 88ca6263..2c05503b 100644 --- a/app/import_utils.py +++ b/app/import_utils.py @@ -25,7 +25,7 @@ def handle_batch_import(batch_import: BatchImport): batch_import.processed = True db.session.commit() - LOG.debug("Start batch import for %s %s", batch_import, user) + LOG.d("Start batch import for %s %s", batch_import, user) file_url = s3.get_url(batch_import.file.path) LOG.d("Download file %s from %s", batch_import.file, file_url) @@ -43,7 +43,7 @@ def import_from_csv(batch_import: BatchImport, user: User, lines): full_alias = sanitize_email(row["alias"]) note = row["note"] except KeyError: - LOG.warning("Cannot parse row %s", row) + LOG.w("Cannot parse row %s", row) continue alias_domain = get_email_domain_part(full_alias) @@ -54,7 +54,7 @@ def import_from_csv(batch_import: BatchImport, user: User, lines): or not custom_domain.verified or custom_domain.user_id != user.id ): - LOG.debug("domain %s can't be used %s", alias_domain, user) + LOG.d("domain %s can't be used %s", alias_domain, user) continue if ( diff --git a/app/jose_utils.py b/app/jose_utils.py index 9118a2f4..7771d536 100644 --- a/app/jose_utils.py +++ b/app/jose_utils.py @@ -62,7 +62,7 @@ def verify_id_token(id_token) -> bool: try: jwt.JWT(key=_key, jwt=id_token) except Exception: - LOG.exception("id token not verified") + LOG.e("id token not verified") return False else: return True diff --git a/app/models.py b/app/models.py index 170e8770..ce78db70 100644 --- a/app/models.py +++ b/app/models.py @@ -685,7 +685,7 @@ class User(db.Model, ModelMixin, UserMixin, PasswordOracle): or not custom_domain.verified or custom_domain.user_id != self.id ): - LOG.warning("Problem with %s default random alias domain", self) + LOG.w("Problem with %s default random alias domain", self) return FIRST_ALIAS_DOMAIN return custom_domain.domain @@ -694,11 +694,11 @@ class User(db.Model, ModelMixin, UserMixin, PasswordOracle): sl_domain = SLDomain.get(self.default_alias_public_domain_id) # sanity check if not sl_domain: - LOG.exception("Problem with %s public random alias domain", self) + LOG.e("Problem with %s public random alias domain", self) return FIRST_ALIAS_DOMAIN if sl_domain.premium_only and not self.is_premium(): - LOG.warning( + LOG.w( "%s is not premium and cannot use %s. Reset default random alias domain setting", self, sl_domain, @@ -863,13 +863,11 @@ def generate_oauth_client_id(client_name) -> str: # check that the client does not exist yet if not Client.get_by(oauth_client_id=oauth_client_id): - LOG.debug("generate oauth_client_id %s", oauth_client_id) + LOG.d("generate oauth_client_id %s", oauth_client_id) return oauth_client_id # Rerun the function - LOG.warning( - "client_id %s already exists, generate a new client_id", oauth_client_id - ) + LOG.w("client_id %s already exists, generate a new client_id", oauth_client_id) return generate_oauth_client_id(client_name) @@ -1041,11 +1039,11 @@ def generate_email( if not Alias.get_by(email=random_email) and not DeletedAlias.get_by( email=random_email ): - LOG.debug("generate email %s", random_email) + LOG.d("generate email %s", random_email) return random_email # Rerun the function - LOG.warning("email %s already exists, generate a new email", random_email) + LOG.w("email %s already exists, generate a new email", random_email) return generate_email(scheme=scheme, in_hex=in_hex) @@ -1240,7 +1238,7 @@ class Alias(db.Model, ModelMixin): elif user.default_alias_public_domain_id: sl_domain: SLDomain = SLDomain.get(user.default_alias_public_domain_id) if sl_domain.premium_only and not user.is_premium(): - LOG.warning("%s not premium, cannot use %s", user, sl_domain) + LOG.w("%s not premium, cannot use %s", user, sl_domain) else: random_email = generate_email( scheme=scheme, in_hex=in_hex, alias_domain=sl_domain.domain @@ -1355,7 +1353,7 @@ class ClientUser(db.Model, ModelMixin): elif scope == Scope.EMAIL: # Use generated email if self.alias_id: - LOG.debug( + LOG.d( "Use gen email for user %s, client %s", self.user, self.client ) res[Scope.EMAIL.value] = self.alias.email @@ -1454,7 +1452,7 @@ class Contact(db.Model, ModelMixin): name, _ = parseaddr_unicode(self.website_from) except Exception: # Skip if website_from is wrongly formatted - LOG.warning( + LOG.w( "Cannot parse contact %s website_from %s", self, self.website_from ) name = "" diff --git a/app/oauth/views/authorize.py b/app/oauth/views/authorize.py index e5ae57ae..b4cc8521 100644 --- a/app/oauth/views/authorize.py +++ b/app/oauth/views/authorize.py @@ -102,7 +102,7 @@ def authorize(): ) user_info = {} if client_user: - LOG.debug("user %s has already allowed client %s", current_user, client) + LOG.d("user %s has already allowed client %s", current_user, client) user_info = client_user.get_user_info() else: suggested_email, other_emails = current_user.suggested_emails( @@ -131,11 +131,11 @@ def authorize(): ) else: # POST - user allows or denies if request.form.get("button") == "deny": - LOG.debug("User %s denies Client %s", current_user, client) + LOG.d("User %s denies Client %s", current_user, client) final_redirect_uri = f"{redirect_uri}?error=deny&state={state}" return redirect(final_redirect_uri) - LOG.debug("User %s allows Client %s", current_user, client) + LOG.d("User %s allows Client %s", current_user, client) client_user = ClientUser.get_by(client_id=client.id, user_id=current_user.id) # user has already allowed this client, user cannot change information @@ -167,11 +167,11 @@ def authorize(): try: alias_suffix = signer.unsign(signed_suffix, max_age=600).decode() except SignatureExpired: - LOG.warning("Alias creation time expired for %s", current_user) + LOG.w("Alias creation time expired for %s", current_user) flash("Alias creation time is expired, please retry", "warning") return redirect(request.url) except Exception: - LOG.warning("Alias suffix is tampered, user %s", current_user) + LOG.w("Alias suffix is tampered, user %s", current_user) flash("Unknown error, refresh the page", "error") return redirect(request.url) @@ -189,7 +189,7 @@ def authorize(): or DeletedAlias.get_by(email=full_alias) or DomainDeletedAlias.get_by(email=full_alias) ): - LOG.exception("alias %s already used, very rare!", full_alias) + LOG.e("alias %s already used, very rare!", full_alias) flash(f"Alias {full_alias} already used", "error") return redirect(request.url) else: @@ -255,9 +255,7 @@ def authorize(): if state: redirect_args["state"] = state else: - LOG.warning( - "more security reason, state should be added. client %s", client - ) + LOG.w("more security reason, state should be added. client %s", client) if scope: redirect_args["scope"] = scope @@ -339,7 +337,7 @@ def generate_access_token() -> str: return access_token # Rerun the function - LOG.warning("access token already exists, generate a new one") + LOG.w("access token already exists, generate a new one") return generate_access_token() diff --git a/app/oauth/views/token.py b/app/oauth/views/token.py index 899b196b..eec1512e 100644 --- a/app/oauth/views/token.py +++ b/app/oauth/views/token.py @@ -56,9 +56,7 @@ def token(): if auth_code.client_id != client.id: return jsonify(error="are you sure this code belongs to you?"), 400 - LOG.debug( - "Create Oauth token for user %s, client %s", auth_code.user, auth_code.client - ) + LOG.d("Create Oauth token for user %s, client %s", auth_code.user, auth_code.client) # Create token oauth_token = OauthToken.create( diff --git a/app/paddle_utils.py b/app/paddle_utils.py index 3b3bb7fd..b2668bc1 100644 --- a/app/paddle_utils.py +++ b/app/paddle_utils.py @@ -72,9 +72,7 @@ def cancel_subscription(subscription_id: str) -> bool: ) res = r.json() if not res["success"]: - LOG.exception( - f"cannot cancel subscription {subscription_id}, paddle response: {res}" - ) + LOG.e(f"cannot cancel subscription {subscription_id}, paddle response: {res}") return res["success"] @@ -102,7 +100,7 @@ def change_plan(user: User, subscription_id: str, plan_id) -> (bool, str): ) return False, "Your card cannot be charged" except KeyError: - LOG.exception( + LOG.e( f"cannot change subscription {subscription_id} to {plan_id}, paddle response: {res}" ) return False, "" diff --git a/app/pgp_utils.py b/app/pgp_utils.py index 1b7003eb..211ec00d 100644 --- a/app/pgp_utils.py +++ b/app/pgp_utils.py @@ -43,7 +43,7 @@ def load_public_key_and_check(public_key: str) -> str: try: encrypt_file(dummy_data, fingerprint) except Exception as e: - LOG.warning( + LOG.w( "Cannot encrypt using the imported key %s %s", fingerprint, public_key ) # remove the fingerprint @@ -55,7 +55,7 @@ def load_public_key_and_check(public_key: str) -> str: def hard_exit(): pid = os.getpid() - LOG.warning("kill pid %s", pid) + LOG.w("kill pid %s", pid) os.kill(pid, 9) diff --git a/app/spamassassin_utils.py b/app/spamassassin_utils.py index b335e9f1..05fb2739 100644 --- a/app/spamassassin_utils.py +++ b/app/spamassassin_utils.py @@ -115,7 +115,7 @@ class SpamAssassin(object): "description": " ".join(wordlist[1:]), } except ValueError: - LOG.warning("Cannot parse %s %s", wordlist[0], wordlist) + LOG.w("Cannot parse %s %s", wordlist[0], wordlist) headers = ( headers.decode("utf-8") diff --git a/cron.py b/cron.py index 273367f2..0a594d14 100644 --- a/cron.py +++ b/cron.py @@ -151,7 +151,7 @@ def notify_manual_sub_end(): if need_reminder: user = manual_sub.user - LOG.debug("Remind user %s that their manual sub is ending soon", user) + LOG.d("Remind user %s that their manual sub is ending soon", user) send_email( user.email, f"Your subscription will end soon", @@ -185,7 +185,7 @@ def notify_manual_sub_end(): if need_reminder: user = coinbase_subscription.user - LOG.debug( + LOG.d( "Remind user %s that their coinbase subscription is ending soon", user ) send_email( @@ -443,9 +443,7 @@ def migrate_domain_trash(): if not SLDomain.get_by(domain=alias_domain): custom_domain = CustomDomain.get_by(domain=alias_domain) if custom_domain: - LOG.exception( - "move %s to domain %s trash", deleted_alias, custom_domain - ) + LOG.e("move %s to domain %s trash", deleted_alias, custom_domain) db.session.add( DomainDeletedAlias( user_id=custom_domain.user_id, @@ -470,7 +468,7 @@ def set_custom_domain_for_alias(): alias_domain = get_email_domain_part(alias.email) custom_domain = CustomDomain.get_by(domain=alias_domain) if custom_domain: - LOG.exception("set %s for %s", custom_domain, alias) + LOG.e("set %s for %s", custom_domain, alias) alias.custom_domain_id = custom_domain.id else: # phantom domain LOG.d("phantom domain %s %s %s", alias.user, alias, alias.enabled) @@ -533,7 +531,7 @@ def sanity_check(): render("transactional/disable-mailbox.html", mailbox=mailbox), ) - LOG.warning( + LOG.w( "issue with mailbox %s domain. #alias %s, nb email log %s", mailbox, mailbox.nb_alias(), @@ -546,40 +544,40 @@ def sanity_check(): for user in User.filter_by(activated=True).all(): if sanitize_email(user.email) != user.email: - LOG.exception("%s does not have sanitized email", user) + LOG.e("%s does not have sanitized email", user) for alias in Alias.query.all(): if sanitize_email(alias.email) != alias.email: - LOG.exception("Alias %s email not sanitized", alias) + LOG.e("Alias %s email not sanitized", alias) if alias.name and "\n" in alias.name: alias.name = alias.name.replace("\n", "") db.session.commit() - LOG.exception("Alias %s name contains linebreak %s", alias, alias.name) + LOG.e("Alias %s name contains linebreak %s", alias, alias.name) contact_email_sanity_date = arrow.get("2021-01-12") for contact in Contact.query.all(): if sanitize_email(contact.reply_email) != contact.reply_email: - LOG.exception("Contact %s reply-email not sanitized", contact) + LOG.e("Contact %s reply-email not sanitized", contact) if ( sanitize_email(contact.website_email) != contact.website_email and contact.created_at > contact_email_sanity_date ): - LOG.exception("Contact %s website-email not sanitized", contact) + LOG.e("Contact %s website-email not sanitized", contact) if not contact.invalid_email and not is_valid_email(contact.website_email): - LOG.exception("%s invalid email", contact) + LOG.e("%s invalid email", contact) contact.invalid_email = True db.session.commit() for mailbox in Mailbox.query.all(): if sanitize_email(mailbox.email) != mailbox.email: - LOG.exception("Mailbox %s address not sanitized", mailbox) + LOG.e("Mailbox %s address not sanitized", mailbox) for contact in Contact.query.all(): if normalize_reply_email(contact.reply_email) != contact.reply_email: - LOG.exception( + LOG.e( "Contact %s reply email is not normalized %s", contact, contact.reply_email, @@ -587,7 +585,7 @@ def sanity_check(): for domain in CustomDomain.query.all(): if domain.name and "\n" in domain.name: - LOG.exception("Domain %s name contain linebreak %s", domain, domain.name) + LOG.e("Domain %s name contain linebreak %s", domain, domain.name) migrate_domain_trash() set_custom_domain_for_alias() @@ -605,7 +603,7 @@ def check_custom_domain(): if sorted(mx_domains) != sorted(EMAIL_SERVERS_WITH_PRIORITY): user = custom_domain.user - LOG.warning( + LOG.w( "The MX record is not correctly set for %s %s %s", custom_domain, user, @@ -617,9 +615,7 @@ def check_custom_domain(): # send alert if fail for 5 consecutive days if custom_domain.nb_failed_checks > 5: domain_dns_url = f"{URL}/dashboard/domains/{custom_domain.id}/dns" - LOG.warning( - "Alert domain MX check fails %s about %s", user, custom_domain - ) + LOG.w("Alert domain MX check fails %s about %s", user, custom_domain) send_email_with_rate_control( user, AlERT_WRONG_MX_RECORD_CUSTOM_DOMAIN, @@ -728,7 +724,7 @@ async def check_hibp(): LOG.d("Checking HIBP API for aliases in breaches") if len(HIBP_API_KEYS) == 0: - LOG.exception("No HIBP API keys") + LOG.e("No HIBP API keys") return LOG.d("Updating list of known breaches") diff --git a/email_handler.py b/email_handler.py index 530a54a1..3d76636f 100644 --- a/email_handler.py +++ b/email_handler.py @@ -347,7 +347,7 @@ def replace_header_when_forward(msg: Message, alias: Alias, header: str): contact.name = contact_name db.session.commit() else: - LOG.debug( + LOG.d( "create contact for alias %s and email %s, header %s", alias, contact_email, @@ -488,7 +488,7 @@ def sign_msg(msg: Message) -> Message: try: signature.set_payload(sign_data(to_bytes(msg).replace(b"\n", b"\r\n"))) except Exception: - LOG.exception("Cannot sign, try using pgpy") + LOG.e("Cannot sign, try using pgpy") signature.set_payload( sign_data_with_pgpy(to_bytes(msg).replace(b"\n", b"\r\n")) ) @@ -871,14 +871,14 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): # Sanity check: verify alias domain is managed by SimpleLogin # scenario: a user have removed a domain but due to a bug, the aliases are still there if not is_valid_alias_address_domain(alias.email): - LOG.exception("%s domain isn't known", alias) + LOG.e("%s domain isn't known", alias) return False, status.E503 user = alias.user mail_from = envelope.mail_from if user.disabled: - LOG.exception( + LOG.e( "User %s disabled, disable sending emails from %s to %s", user, alias, @@ -1094,7 +1094,7 @@ def get_mailbox_from_mail_from(mail_from: str, alias) -> Optional[Mailbox]: for address in mailbox.authorized_addresses: if address.email == mail_from: - LOG.debug( + LOG.d( "Found an authorized address for %s %s %s", alias, mailbox, address ) return mailbox @@ -1172,12 +1172,12 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): # email_log.mailbox should be set during the forward phase if not mailbox: - LOG.exception("Use %s default mailbox %s", alias, alias.mailbox) + LOG.e("Use %s default mailbox %s", alias, alias.mailbox) mailbox = alias.mailbox Bounce.create(email=mailbox.email, commit=True) - LOG.debug( + LOG.d( "Handle forward bounce %s -> %s -> %s. %s", contact, alias, mailbox, email_log ) @@ -1325,9 +1325,7 @@ def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog): user = alias.user mailbox = email_log.mailbox or alias.mailbox - LOG.debug( - "Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log - ) + LOG.d("Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log) Bounce.create(email=sanitize_email(contact.website_email), commit=True) @@ -1548,11 +1546,11 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str: """return the SMTP status""" user = User.get(user_id) if not user: - LOG.exception("No such user %s %s", user_id, mail_from) + LOG.e("No such user %s %s", user_id, mail_from) return status.E510 if mail_from != user.email: - LOG.exception("Unauthorized mail_from %s %s", user, mail_from) + LOG.e("Unauthorized mail_from %s %s", user, mail_from) return status.E511 user.notification = False diff --git a/init_app.py b/init_app.py index a5e41a1c..4c13e15e 100644 --- a/init_app.py +++ b/init_app.py @@ -15,9 +15,7 @@ def load_pgp_public_keys(): # sanity check if fingerprint != mailbox.pgp_finger_print: - LOG.exception( - "fingerprint %s different for mailbox %s", fingerprint, mailbox - ) + LOG.e("fingerprint %s different for mailbox %s", fingerprint, mailbox) mailbox.pgp_finger_print = fingerprint db.session.commit() @@ -27,9 +25,7 @@ def load_pgp_public_keys(): # sanity check if fingerprint != contact.pgp_finger_print: - LOG.exception( - "fingerprint %s different for contact %s", fingerprint, contact - ) + LOG.e("fingerprint %s different for contact %s", fingerprint, contact) contact.pgp_finger_print = fingerprint db.session.commit() @@ -42,14 +38,14 @@ def add_sl_domains(): if SLDomain.get_by(domain=alias_domain): LOG.d("%s is already a SL domain", alias_domain) else: - LOG.info("Add %s to SL domain", alias_domain) + LOG.i("Add %s to SL domain", alias_domain) SLDomain.create(domain=alias_domain) for premium_domain in PREMIUM_ALIAS_DOMAINS: if SLDomain.get_by(domain=premium_domain): LOG.d("%s is already a SL domain", premium_domain) else: - LOG.info("Add %s to SL domain", premium_domain) + LOG.i("Add %s to SL domain", premium_domain) SLDomain.create(domain=premium_domain, premium_only=True) db.session.commit() diff --git a/job_runner.py b/job_runner.py index 74dbf082..06c12487 100644 --- a/job_runner.py +++ b/job_runner.py @@ -159,7 +159,7 @@ if __name__ == "__main__": continue user_email = user.email - LOG.warning("Delete user %s", user) + LOG.w("Delete user %s", user) User.delete(user.id) db.session.commit() @@ -170,6 +170,6 @@ if __name__ == "__main__": render("transactional/account-delete.html"), ) else: - LOG.exception("Unknown job name %s", job.name) + LOG.e("Unknown job name %s", job.name) time.sleep(10) diff --git a/monitoring.py b/monitoring.py index 74dd8e3d..ff92f672 100644 --- a/monitoring.py +++ b/monitoring.py @@ -43,7 +43,7 @@ def get_stats(): # reset _nb_failed = 0 - LOG.exception( + LOG.e( "Too many emails in incoming & active queue %s %s", incoming_queue, active_queue, diff --git a/server.py b/server.py index 8726bb20..3c7ef5a6 100644 --- a/server.py +++ b/server.py @@ -485,7 +485,7 @@ def set_index_page(app): and not request.path.startswith("/admin/static") and not request.path.startswith("/_debug_toolbar") ): - LOG.debug( + LOG.d( "%s %s %s %s %s, takes %s", request.remote_addr, request.method, @@ -563,7 +563,7 @@ def setup_error_page(app): @app.errorhandler(429) def rate_limited(e): - LOG.warning( + LOG.w( "Client hit rate limit on path %s, user:%s", request.path, get_current_user(), @@ -589,7 +589,7 @@ def setup_error_page(app): @app.errorhandler(Exception) def error_handler(e): - LOG.exception(e) + LOG.e(e) if request.path.startswith("/api/"): return jsonify(error="Internal error"), 500 else: @@ -632,13 +632,11 @@ def jinja2_filter(app): def setup_paddle_callback(app: Flask): @app.route("/paddle", methods=["GET", "POST"]) def paddle(): - LOG.debug(f"paddle callback {request.form.get('alert_name')} {request.form}") + LOG.d(f"paddle callback {request.form.get('alert_name')} {request.form}") # make sure the request comes from Paddle if not paddle_utils.verify_incoming_request(dict(request.form)): - LOG.exception( - "request not coming from paddle. Request data:%s", dict(request.form) - ) + LOG.e("request not coming from paddle. Request data:%s", dict(request.form)) return "KO", 400 if ( @@ -657,7 +655,7 @@ def setup_paddle_callback(app: Flask): elif subscription_plan_id in PADDLE_YEARLY_PRODUCT_IDS: plan = PlanEnum.yearly else: - LOG.exception( + LOG.e( "Unknown subscription_plan_id %s %s", subscription_plan_id, request.form, @@ -694,13 +692,13 @@ def setup_paddle_callback(app: Flask): # in case user cancels a plan and subscribes a new plan sub.cancelled = False - LOG.debug("User %s upgrades!", user) + LOG.d("User %s upgrades!", user) db.session.commit() elif request.form.get("alert_name") == "subscription_payment_succeeded": subscription_id = request.form.get("subscription_id") - LOG.debug("Update subscription %s", subscription_id) + LOG.d("Update subscription %s", subscription_id) sub: Subscription = Subscription.get_by(subscription_id=subscription_id) # when user subscribes, the "subscription_payment_succeeded" can arrive BEFORE "subscription_created" @@ -719,7 +717,7 @@ def setup_paddle_callback(app: Flask): sub: Subscription = Subscription.get_by(subscription_id=subscription_id) if sub: # cancellation_effective_date should be the same as next_bill_date - LOG.warning( + LOG.w( "Cancel subscription %s %s on %s, next bill date %s", subscription_id, sub.user, @@ -749,7 +747,7 @@ def setup_paddle_callback(app: Flask): sub: Subscription = Subscription.get_by(subscription_id=subscription_id) if sub: - LOG.debug( + LOG.d( "Update subscription %s %s on %s, next bill date %s", subscription_id, sub.user, @@ -795,7 +793,7 @@ def setup_coinbase_commerce(app): request_data, request_sig, COINBASE_WEBHOOK_SECRET ) except (WebhookInvalidPayload, SignatureVerificationError) as e: - LOG.exception("Invalid Coinbase webhook") + LOG.e("Invalid Coinbase webhook") return str(e), 400 LOG.d("Coinbase event %s", event) @@ -814,7 +812,7 @@ def handle_coinbase_event(event) -> bool: code = event["data"]["code"] user = User.get(user_id) if not user: - LOG.exception("User not found %s", user_id) + LOG.e("User not found %s", user_id) return False coinbase_subscription: CoinbaseSubscription = CoinbaseSubscription.get_by( @@ -922,7 +920,7 @@ def register_custom_commands(app): def dummy_data(): from init_app import add_sl_domains - LOG.warning("reset db, add fake data") + LOG.w("reset db, add fake data") with app.app_context(): fake_data() add_sl_domains() diff --git a/shell.py b/shell.py index 6cee5fec..d3df9c1e 100644 --- a/shell.py +++ b/shell.py @@ -23,7 +23,7 @@ from server import create_app def create_db(): if not database_exists(DB_URI): - LOG.debug("db not exist, create database") + LOG.d("db not exist, create database") create_database(DB_URI) # Create all tables @@ -56,7 +56,7 @@ def send_mailbox_newsletter(): ) sleep(1) except Exception: - LOG.warning("Cannot send to user %s", user) + LOG.w("Cannot send to user %s", user) def send_pgp_newsletter(): @@ -72,7 +72,7 @@ def send_pgp_newsletter(): ) sleep(1) except Exception: - LOG.warning("Cannot send to user %s", user) + LOG.w("Cannot send to user %s", user) def send_mobile_newsletter(): @@ -89,7 +89,7 @@ def send_mobile_newsletter(): render("com/newsletter/mobile-darkmode.html", user=user), ) except Exception: - LOG.warning("Cannot send to user %s", user) + LOG.w("Cannot send to user %s", user) if count % 5 == 0: # sleep every 5 sends to avoid hitting email limits @@ -125,7 +125,7 @@ def disable_mailbox(mailbox_id): email_msg.replace("\n", "
"), ) except Exception: - LOG.exception("Cannot send disable mailbox email to %s", mailbox.user) + LOG.e("Cannot send disable mailbox email to %s", mailbox.user) def send_onboarding_emails(user): From 4df83f953d9d268e595d791de7524256c8dd7071 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 15:17:11 +0200 Subject: [PATCH 046/144] handle utf-8 decoding fail --- app/email_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/email_utils.py b/app/email_utils.py index 4eefbb6d..80153606 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -699,7 +699,12 @@ def parseaddr_unicode(addr) -> (str, str): LOG.w( "Cannot decode %s with %s, use utf-8", decoded_string, charset ) - name = decoded_string.decode("utf-8") + try: + name = decoded_string.decode("utf-8") + except UnicodeDecodeError: + LOG.w("utf-8 not work on %s", decoded_string) + name = "" + else: name = decoded_string From a58cf9dd5e9c4f5dba06c5c8ffea63e0ad71bc1d Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 15:23:48 +0200 Subject: [PATCH 047/144] use warning level for log --- email_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 3d76636f..f1b8d8f0 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1296,7 +1296,7 @@ def handle_hotmail_complaint(msg: Message): return user = alias.user - LOG.e("Handle hotmail complaint for %s %s", alias, user) + LOG.w("Handle hotmail complaint for %s %s %s", alias, user, alias.mailboxes) send_email_with_rate_control( user, From ff2cbeb3af85dda4574ba53448e15e6ba09aece5 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 15:49:47 +0200 Subject: [PATCH 048/144] handle case to header isn't present --- email_handler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/email_handler.py b/email_handler.py index f1b8d8f0..9c57b4b5 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1288,6 +1288,11 @@ def handle_hotmail_complaint(msg: Message): """ orig_msg = get_orig_message_from_outlook_complaint(msg) to_header = orig_msg["To"] + if not to_header: + LOG.e("cannot find the alias") + LOG.d("msg:%s, \n orig_msg:%s", msg, orig_msg) + return + _, alias_address = parseaddr_unicode(to_header) alias = Alias.get_by(email=alias_address) From 493a5daa45612649bb3dda3d81a4a1d67d1be472 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 8 Sep 2021 18:25:40 +0200 Subject: [PATCH 049/144] use warning level --- email_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 9c57b4b5..310cd36f 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1704,7 +1704,7 @@ def handle(envelope: Envelope) -> str: contact = Contact.get_by(reply_email=mail_from) if contact: - LOG.e( + LOG.w( "email can't be sent from a reverse-alias:%s, contact email:%s, %s, %s", contact.reply_email, contact.website_email, From defd7b159dcdd37733ba0bdbcfc6ac944afe33c3 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Thu, 9 Sep 2021 11:47:01 +0200 Subject: [PATCH 050/144] Fix get_header_unicode: handle the case header contains several parts --- app/email_utils.py | 35 ++++++++++++++++++++++------------- tests/test_email_utils.py | 6 ++++++ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 80153606..606d9af3 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -652,24 +652,33 @@ def get_spam_from_header(spam_status_header, max_score=None) -> (bool, str): def get_header_unicode(header: str) -> str: + """ + Convert a header to unicode + Should be used to handle headers like From:, To:, CC:, Subject: + """ if header is None: return "" - decoded_string, charset = decode_header(header)[0] - if charset is not None: - try: - return decoded_string.decode(charset) - except UnicodeDecodeError: - LOG.w("Cannot decode header %s", header) - except LookupError: # charset is unknown - LOG.w("Cannot decode %s with %s, use utf-8", decoded_string, charset) + ret = "" + for to_decoded_str, charset in decode_header(header): + if charset is None: + if type(to_decoded_str) is bytes: + decoded_str = to_decoded_str.decode() + else: + decoded_str = to_decoded_str + else: try: - return decoded_string.decode("utf-8") - except UnicodeDecodeError: - LOG.w("Cannot UTF-8 decode %s", decoded_string) - return decoded_string.decode("utf-8", errors="replace") + decoded_str = to_decoded_str.decode(charset) + except (LookupError, UnicodeDecodeError): # charset is unknown + LOG.w("Cannot decode %s with %s, try utf-8", to_decoded_str, charset) + try: + decoded_str = to_decoded_str.decode("utf-8") + except UnicodeDecodeError: + LOG.w("Cannot UTF-8 decode %s", to_decoded_str) + decoded_str = to_decoded_str.decode("utf-8", errors="replace") + ret += decoded_str - return header + return ret def parseaddr_unicode(addr) -> (str, str): diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index 5f432356..5e1e7528 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -29,6 +29,7 @@ from app.email_utils import ( parse_id_from_bounce, get_queue_id, should_ignore_bounce, + get_header_unicode, ) from app.extensions import db from app.models import User, CustomDomain, Alias, Contact, EmailLog, IgnoreBounceSender @@ -748,3 +749,8 @@ def test_should_ignore_bounce(flask_client): IgnoreBounceSender.create(mail_from="to-ignore@example.com") assert should_ignore_bounce("to-ignore@example.com") + + +def test_get_header_unicode(): + assert get_header_unicode("ab@cd.com") == "ab@cd.com" + assert get_header_unicode("=?utf-8?B?w6nDqQ==?=@example.com") == "éé@example.com" From db55f289c1d84f1c843618b359ea716c09a74a3b Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Thu, 9 Sep 2021 11:47:14 +0200 Subject: [PATCH 051/144] upgrade aiosmtpd to 1.4.2 --- poetry.lock | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7590df12..ff3d4640 100644 --- a/poetry.lock +++ b/poetry.lock @@ -18,14 +18,16 @@ speedups = ["aiodns", "brotlipy", "cchardet"] [[package]] name = "aiosmtpd" -version = "1.2" +version = "1.4.2" description = "aiosmtpd - asyncio based SMTP server" category = "main" optional = false -python-versions = "*" +python-versions = "~=3.6" [package.dependencies] atpublic = "*" +attrs = "*" +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "aiosmtplib" @@ -1769,7 +1771,8 @@ aiohttp = [ {file = "aiohttp-3.6.2.tar.gz", hash = "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326"}, ] aiosmtpd = [ - {file = "aiosmtpd-1.2.tar.gz", hash = "sha256:b7ea7ee663f3b8514d3224d55c4e8827148277b124ea862a0bbfca1bc899aef5"}, + {file = "aiosmtpd-1.4.2-py3-none-any.whl", hash = "sha256:314f70b74cb8474882cef396b186fbfad8660c7b52be5c1937f3c31df14232a4"}, + {file = "aiosmtpd-1.4.2.tar.gz", hash = "sha256:aa891d010d2097274189078c6ce2a59a167f3fb2e974e028b572a61e92e1549c"}, ] aiosmtplib = [ {file = "aiosmtplib-1.1.4-py3-none-any.whl", hash = "sha256:93e53edac183f1a608bc34464efeef23902e59e949017b1682014f59ecdcd37d"}, From 6e9dfdd6f12a483d79a5bff6a82f3a51fac6a073 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Thu, 9 Sep 2021 18:54:54 +0200 Subject: [PATCH 052/144] if the complaint cannot be handled, forward it normally --- email_handler.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/email_handler.py b/email_handler.py index 310cd36f..f745a9c8 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1282,23 +1282,23 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): ) -def handle_hotmail_complaint(msg: Message): +def handle_hotmail_complaint(msg: Message) -> bool: """ Handle hotmail complaint sent to postmaster + Return True if the complaint can be handled, False otherwise """ orig_msg = get_orig_message_from_outlook_complaint(msg) to_header = orig_msg["To"] if not to_header: LOG.e("cannot find the alias") - LOG.d("msg:%s, \n orig_msg:%s", msg, orig_msg) - return + return False _, alias_address = parseaddr_unicode(to_header) alias = Alias.get_by(email=alias_address) if not alias: LOG.d("No alias for %s", alias_address) - return + return False user = alias.user LOG.w("Handle hotmail complaint for %s %s %s", alias, user, alias.mailboxes) @@ -1319,6 +1319,8 @@ def handle_hotmail_complaint(msg: Message): max_nb_alert=2, ) + return True + def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog): """ @@ -1734,8 +1736,10 @@ def handle(envelope: Envelope) -> str: and rcpt_tos[0] == POSTMASTER ): LOG.w("Handle hotmail complaint") - handle_hotmail_complaint(msg) - return status.E208 + + # if the complaint cannot be handled, forward it normally + if handle_hotmail_complaint(msg): + return status.E208 # Handle bounce if ( From 6b65e00dcfc65c85ad05b1d0085a6d64483f10ce Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 16:36:59 +0200 Subject: [PATCH 053/144] install flanker, upgrade email_validator --- app/email_utils.py | 2 +- poetry.lock | 84 ++++++++++++++++++++++++++++++++++++++++++---- pyproject.toml | 1 + 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 606d9af3..2f49b728 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -651,7 +651,7 @@ def get_spam_from_header(spam_status_header, max_score=None) -> (bool, str): return spamassassin_answer.lower() == "yes", spam_status_header -def get_header_unicode(header: str) -> str: +def get_header_unicode(header: Union[str, Header]) -> str: """ Convert a header to unicode Should be used to handle headers like From:, To:, CC:, Subject: diff --git a/poetry.lock b/poetry.lock index ff3d4640..81b099cd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -398,11 +398,11 @@ trio = ["trio (>=0.14.0)", "sniffio (>=1.1)"] [[package]] name = "email-validator" -version = "1.1.1" +version = "1.1.3" description = "A robust email syntax and deliverability validation library for Python 2.x/3.x." category = "main" optional = false -python-versions = "*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] dnspython = ">=1.15.0" @@ -441,6 +441,30 @@ mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.6.0a1,<2.7.0" pyflakes = ">=2.2.0,<2.3.0" +[[package]] +name = "flanker" +version = "0.9.11" +description = "Mailgun Parsing Tools" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +attrs = "*" +chardet = ">=1.0.1" +cryptography = ">=0.5" +idna = ">=2.5" +ply = ">=3.10" +regex = ">=0.1.20110315" +six = "*" +tld = "*" +WebOb = ">=0.9.8" + +[package.extras] +cchardet = ["cchardet (>=0.3.5)"] +tests = ["coverage", "coveralls", "mock", "nose"] +validator = ["dnsq (>=1.1.6)", "redis (>=2.7.1)"] + [[package]] name = "flask" version = "1.1.2" @@ -1060,6 +1084,14 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] +[[package]] +name = "ply" +version = "3.11" +description = "Python Lex & Yacc" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "pre-commit" version = "2.7.1" @@ -1350,7 +1382,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" name = "regex" version = "2020.9.27" description = "Alternative regular expression module, to replace re." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -1535,6 +1567,14 @@ python-versions = "*" python-dateutil = ">=2.6.0" "ruamel.yaml" = ">=0.14.2" +[[package]] +name = "tld" +version = "0.12.6" +description = "Extract the top-level domain (TLD) from the URL given." +category = "main" +optional = false +python-versions = ">=2.7, <4" + [[package]] name = "toml" version = "0.10.1" @@ -1655,6 +1695,18 @@ future = ">=0.17.1" pyOpenSSL = ">=16.0.0" six = ">=1.11.0" +[[package]] +name = "webob" +version = "1.8.7" +description = "WSGI request and response object" +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*" + +[package.extras] +docs = ["Sphinx (>=1.7.5)", "pylons-sphinx-themes"] +testing = ["pytest (>=3.1.0)", "coverage", "pytest-cov", "pytest-xdist"] + [[package]] name = "werkzeug" version = "1.0.1" @@ -1753,7 +1805,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "6a909880eb2c4afcd161158bd606edde3839aedbfca57a751c6b0c6e5306d9dc" +content-hash = "a80a9f350a39e926d74ea73a7b9d5b04ec8b515bcc2e9bc31bd2537989763aa5" [metadata.files] aiohttp = [ @@ -1992,8 +2044,8 @@ dnspython = [ {file = "dnspython-2.0.0.zip", hash = "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7"}, ] email-validator = [ - {file = "email_validator-1.1.1-py2.py3-none-any.whl", hash = "sha256:5f246ae8d81ce3000eade06595b7bb55a4cf350d559e890182a1466a21f25067"}, - {file = "email_validator-1.1.1.tar.gz", hash = "sha256:63094045c3e802c3d3d575b18b004a531c36243ca8d1cec785ff6bfcb04185bb"}, + {file = "email_validator-1.1.3-py2.py3-none-any.whl", hash = "sha256:5675c8ceb7106a37e40e2698a57c056756bf3f272cfa8682a4f87ebd95d8440b"}, + {file = "email_validator-1.1.3.tar.gz", hash = "sha256:aa237a65f6f4da067119b7df3f13e89c25c051327b2b5b66dc075f33d62480d7"}, ] facebook-sdk = [ {file = "facebook-sdk-3.1.0.tar.gz", hash = "sha256:cabcd2e69ea3d9f042919c99b353df7aa1e2be86d040121f6e9f5e63c1cf0f8d"}, @@ -2007,6 +2059,9 @@ flake8 = [ {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"}, {file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"}, ] +flanker = [ + {file = "flanker-0.9.11.tar.gz", hash = "sha256:974418e5b498fd3bcb3859c22e22d26495257f9cf98b744c17f2335aca86e001"}, +] flask = [ {file = "Flask-1.1.2-py2.py3-none-any.whl", hash = "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557"}, {file = "Flask-1.1.2.tar.gz", hash = "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060"}, @@ -2298,6 +2353,10 @@ pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] +ply = [ + {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, + {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, +] pre-commit = [ {file = "pre_commit-2.7.1-py2.py3-none-any.whl", hash = "sha256:810aef2a2ba4f31eed1941fc270e72696a1ad5590b9751839c90807d0fff6b9a"}, {file = "pre_commit-2.7.1.tar.gz", hash = "sha256:c54fd3e574565fe128ecc5e7d2f91279772ddb03f8729645fa812fe809084a70"}, @@ -2694,6 +2753,15 @@ sqlalchemy-utils = [ strictyaml = [ {file = "strictyaml-1.1.0.tar.gz", hash = "sha256:6b07dbd4f77ab023ed4167c43ffc1b9f9354fb6075cc6ff3b91fefcbb80342ca"}, ] +tld = [ + {file = "tld-0.12.6-py27-none-any.whl", hash = "sha256:ef5b162d6fa295822dacd4fe4df1b62d8df2550795a97399a8905821b58d3702"}, + {file = "tld-0.12.6-py35-none-any.whl", hash = "sha256:826bbe61dccc8d63144b51caef83e1373fbaac6f9ada46fca7846021f5d36fef"}, + {file = "tld-0.12.6-py36-none-any.whl", hash = "sha256:843844e4256c943983d86366b5af3ac9cd1c9a0b6465f04d9f70e3b4c1a7989f"}, + {file = "tld-0.12.6-py37-none-any.whl", hash = "sha256:a92ac6b84917e7d9e934434b8d37e9be534598f138fbb86b3c0d5426f2621890"}, + {file = "tld-0.12.6-py38-none-any.whl", hash = "sha256:266106ad9035f54cd5cce5f823911a51f697e7c58cb45bfbd6c53b4c2976ece2"}, + {file = "tld-0.12.6-py39-none-any.whl", hash = "sha256:b6650f2d5392a49760064bc55d73ce3397a378ef24ded96efb516c6b8ec68c26"}, + {file = "tld-0.12.6.tar.gz", hash = "sha256:69fed19d26bb3f715366fb4af66fdeace896c55c052b00e8aaba3a7b63f3e7f0"}, +] toml = [ {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"}, @@ -2758,6 +2826,10 @@ webauthn = [ {file = "webauthn-0.4.7-py2.py3-none-any.whl", hash = "sha256:238391b2e2cc60fb51a2cd2d2d6be149920b9af6184651353d9f95856617a9e7"}, {file = "webauthn-0.4.7.tar.gz", hash = "sha256:8ad9072ff1d6169f3be30d4dc8733ea563dd266962397bc58b40f674a6af74ac"}, ] +webob = [ + {file = "WebOb-1.8.7-py2.py3-none-any.whl", hash = "sha256:73aae30359291c14fa3b956f8b5ca31960e420c28c1bec002547fb04928cf89b"}, + {file = "WebOb-1.8.7.tar.gz", hash = "sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323"}, +] werkzeug = [ {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, {file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"}, diff --git a/pyproject.toml b/pyproject.toml index ce732aed..6c38e8d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ py3-validate-email = "^0.2.10" coinbase-commerce = "^1.0.1" requests = "^2.25.1" newrelic = "^6.4.4" +flanker = "^0.9.11" [tool.poetry.dev-dependencies] pytest = "^6.1.0" From 8023afe9bef4fcfb5870d6ac00bc3547b0afa482 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 16:38:59 +0200 Subject: [PATCH 054/144] use email_validator instead of validate_email which isn't updated for a while --- app/email_utils.py | 22 ++++++++++++++-------- poetry.lock | 20 ++------------------ pyproject.toml | 1 - 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 2f49b728..a4e8af5c 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -6,22 +6,22 @@ import quopri import random import re import time +from copy import deepcopy from email.errors import HeaderParseError -from email.header import decode_header +from email.header import decode_header, Header from email.message import Message from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import make_msgid, formatdate, parseaddr from smtplib import SMTP, SMTPServerDisconnected -from typing import Tuple, List, Optional -from copy import deepcopy +from typing import Tuple, List, Optional, Union import arrow import dkim import spf +from email_validator import validate_email, EmailNotValidError from jinja2 import Environment, FileSystemLoader from sqlalchemy import func -from validate_email import validate_email from app.config import ( SUPPORT_EMAIL, @@ -769,10 +769,16 @@ def should_add_dkim_signature(domain: str) -> bool: def is_valid_email(email_address: str) -> bool: - """Used to check whether an email address is valid""" - return validate_email( - email_address=email_address, check_mx=False, use_blacklist=False - ) + """ + Used to check whether an email address is valid + NOT run MX check. + NOT allow unicode. + """ + try: + validate_email(email_address, check_deliverability=False, allow_smtputf8=False) + return True + except EmailNotValidError: + return False class EmailEncoding(enum.Enum): diff --git a/poetry.lock b/poetry.lock index 81b099cd..199d9e32 100644 --- a/poetry.lock +++ b/poetry.lock @@ -423,7 +423,7 @@ requests = "*" name = "filelock" version = "3.0.12" description = "A platform independent file lock." -category = "main" +category = "dev" optional = false python-versions = "*" @@ -1166,19 +1166,6 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "py3-validate-email" -version = "0.2.10" -description = "Email validator with regex, blacklisted domains and SMTP checking." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -dnspython = ">=2.0,<3.0" -filelock = ">=3.0,<4.0" -idna = ">=2.10,<3.0" - [[package]] name = "pyasn1" version = "0.4.8" @@ -1805,7 +1792,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "a80a9f350a39e926d74ea73a7b9d5b04ec8b515bcc2e9bc31bd2537989763aa5" +content-hash = "a2f1de1d4ecbe4b1e36d965bb9ba52d10d16b344b6c7e4509a53245b67fdcec5" [metadata.files] aiohttp = [ @@ -2440,9 +2427,6 @@ py = [ {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, ] -py3-validate-email = [ - {file = "py3-validate-email-0.2.10.tar.gz", hash = "sha256:3bbb264b49c0ae09afdb2738956f00b0e8dd7e079e2d079b2e9b6688de474d28"}, -] pyasn1 = [ {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, diff --git a/pyproject.toml b/pyproject.toml index 6c38e8d3..9c3e46c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,6 @@ gevent = "^20.9.0" aiospamc = "^0.6.1" email_validator = "^1.1.1" PGPy = "^0.5.3" -py3-validate-email = "^0.2.10" coinbase-commerce = "^1.0.1" requests = "^2.25.1" newrelic = "^6.4.4" From 500ff00c7c778ab7f70285e08cb6505fd1c54525 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 16:51:36 +0200 Subject: [PATCH 055/144] use flanker to parse To:, CC: header in replace_header_when_forward() --- email_handler.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/email_handler.py b/email_handler.py index f745a9c8..1b22c27f 100644 --- a/email_handler.py +++ b/email_handler.py @@ -47,6 +47,9 @@ from typing import List, Tuple, Optional import newrelic.agent from aiosmtpd.controller import Controller from aiosmtpd.smtp import Envelope +from email_validator import validate_email, EmailNotValidError +from flanker.addresslib import address +from flanker.addresslib.address import EmailAddress from sqlalchemy.exc import IntegrityError from app import pgp_utils, s3 @@ -317,34 +320,38 @@ def replace_header_when_forward(msg: Message, alias: Alias, header: str): new_addrs: [str] = [] headers = msg.get_all(header, []) # headers can be an array of Header, convert it to string here - headers = [str(h) for h in headers] - for contact_name, contact_email in getaddresses(headers): - # convert back to original then parse again to make sure contact_name is unicode - addr = formataddr((contact_name, contact_email)) - contact_name, _ = parseaddr_unicode(addr) + headers = [get_header_unicode(h) for h in headers] - contact_email = sanitize_email(contact_email) + full_addresses: [EmailAddress] = [] + for h in headers: + full_addresses += address.parse_list(h) + + for full_address in full_addresses: + contact_email = sanitize_email(full_address.address) # no transformation when alias is already in the header if contact_email == alias.email: - new_addrs.append(addr) + new_addrs.append(full_address.full_spec()) continue - if not is_valid_email(contact_email): + try: + # NOT allow unicode for contact address + validate_email(contact_email, check_deliverability=False, allow_smtputf8=False) + except EmailNotValidError: LOG.w("invalid contact email %s. %s. Skip", contact_email, headers) continue contact = Contact.get_by(alias_id=alias.id, website_email=contact_email) if contact: # update the contact name if needed - if contact.name != contact_name: + if contact.name != full_address.display_name: LOG.d( "Update contact %s name %s to %s", contact, contact.name, - contact_name, + full_address.display_name, ) - contact.name = contact_name + contact.name = full_address.display_name db.session.commit() else: LOG.d( @@ -359,10 +366,9 @@ def replace_header_when_forward(msg: Message, alias: Alias, header: str): user_id=alias.user_id, alias_id=alias.id, website_email=contact_email, - name=contact_name, + name=full_address.display_name, reply_email=generate_reply_email(contact_email, alias.user), is_cc=header.lower() == "cc", - from_header=addr, ) db.session.commit() except IntegrityError: From 3ad4b6b76f00101ae187390f4625dc4c189ce0d6 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 17:06:38 +0200 Subject: [PATCH 056/144] use flanker instead of parseaddr_unicode --- app/api/views/alias.py | 6 ++- app/dashboard/views/alias_contact_manager.py | 9 +++- app/models.py | 7 ++-- email_handler.py | 43 +++++++++++--------- 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/app/api/views/alias.py b/app/api/views/alias.py index efe6f9dc..b1f78b78 100644 --- a/app/api/views/alias.py +++ b/app/api/views/alias.py @@ -1,3 +1,5 @@ +from flanker.addresslib import address +from flanker.addresslib.address import EmailAddress from flask import g from flask import jsonify from flask import request @@ -16,7 +18,6 @@ from app.api.serializer import ( ) from app.dashboard.views.alias_log import get_alias_log from app.email_utils import ( - parseaddr_unicode, is_valid_email, generate_reply_email, ) @@ -400,7 +401,8 @@ def create_contact_route(alias_id): if not contact_addr: return jsonify(error="Contact cannot be empty"), 400 - contact_name, contact_email = parseaddr_unicode(contact_addr) + full_address: EmailAddress = address.parse(contact_addr) + contact_name, contact_email = full_address.display_name, full_address.address if not is_valid_email(contact_email): return jsonify(error=f"invalid contact email {contact_email}"), 400 diff --git a/app/dashboard/views/alias_contact_manager.py b/app/dashboard/views/alias_contact_manager.py index e34a0110..6d80f5b5 100644 --- a/app/dashboard/views/alias_contact_manager.py +++ b/app/dashboard/views/alias_contact_manager.py @@ -1,6 +1,8 @@ from dataclasses import dataclass from operator import or_ +from flanker.addresslib import address +from flanker.addresslib.address import EmailAddress from flask import render_template, request, redirect, flash from flask import url_for from flask_login import login_required, current_user @@ -11,7 +13,6 @@ from wtforms import StringField, validators, ValidationError from app.config import PAGE_LIMIT from app.dashboard.base import dashboard_bp from app.email_utils import ( - parseaddr_unicode, is_valid_email, generate_reply_email, ) @@ -182,7 +183,11 @@ def alias_contact_manager(alias_id): contact_addr = new_contact_form.email.data.strip() try: - contact_name, contact_email = parseaddr_unicode(contact_addr) + full_address: EmailAddress = address.parse(contact_addr) + contact_name, contact_email = ( + full_address.display_name, + full_address.address, + ) contact_email = sanitize_email(contact_email) except Exception: flash(f"{contact_addr} is invalid", "error") diff --git a/app/models.py b/app/models.py index ce78db70..6997b687 100644 --- a/app/models.py +++ b/app/models.py @@ -7,6 +7,7 @@ from typing import List, Tuple, Optional import arrow import sqlalchemy as sa from arrow import Arrow +from flanker.addresslib import address from flask import url_for from flask_login import UserMixin from sqlalchemy import text, desc, CheckConstraint, Index, Column @@ -1447,12 +1448,10 @@ class Contact(db.Model, ModelMixin): # if no name, try to parse it from website_from if not name and self.website_from: try: - from app.email_utils import parseaddr_unicode - - name, _ = parseaddr_unicode(self.website_from) + name = address.parse(self.website_from).display_name except Exception: # Skip if website_from is wrongly formatted - LOG.w( + LOG.e( "Cannot parse contact %s website_from %s", self, self.website_from ) name = "" diff --git a/email_handler.py b/email_handler.py index 1b22c27f..bcfceab7 100644 --- a/email_handler.py +++ b/email_handler.py @@ -95,7 +95,6 @@ from app.email_utils import ( delete_all_headers_except, get_spam_info, get_orig_message_from_spamassassin_report, - parseaddr_unicode, send_email_with_rate_control, get_email_domain_part, copy, @@ -180,16 +179,17 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con """ contact_from_header is the RFC 2047 format FROM header """ - contact_name, contact_email = parseaddr_unicode(from_header) + full_address: EmailAddress = address.parse(from_header) + contact_name, contact_email = full_address.display_name, full_address.address if not is_valid_email(contact_email): # From header is wrongly formatted, try with mail_from if mail_from and mail_from != "<>": LOG.w( - "Cannot parse email from from_header %s, parse from mail_from %s", + "Cannot parse email from from_header %s, use mail_from %s", from_header, mail_from, ) - _, contact_email = parseaddr_unicode(mail_from) + contact_email = mail_from if not is_valid_email(contact_email): LOG.w( @@ -273,25 +273,23 @@ def get_or_create_reply_to_contact( """ Get or create the contact for the Reply-To header """ - name, address = parseaddr_unicode(reply_to_header) + full_address: EmailAddress = address.parse(reply_to_header) - if not is_valid_email(address): + if not is_valid_email(full_address.address): LOG.w( "invalid reply-to address %s. Parse from %s", - address, + full_address, reply_to_header, ) return None - address = sanitize_email(address) - - contact = Contact.get_by(alias_id=alias.id, website_email=address) + contact = Contact.get_by(alias_id=alias.id, website_email=full_address.address) if contact: return contact else: LOG.d( "create contact %s for alias %s via reply-to header", - address, + full_address.address, alias, reply_to_header, ) @@ -300,15 +298,17 @@ def get_or_create_reply_to_contact( contact = Contact.create( user_id=alias.user_id, alias_id=alias.id, - website_email=address, - name=name, - reply_email=generate_reply_email(address, alias.user), + website_email=full_address.address, + name=full_address.display_name, + reply_email=generate_reply_email(full_address.address, alias.user), ) db.session.commit() except IntegrityError: - LOG.w("Contact %s %s already exist", alias, address) + LOG.w("Contact %s %s already exist", alias, full_address.address) db.session.rollback() - contact = Contact.get_by(alias_id=alias.id, website_email=address) + contact = Contact.get_by( + alias_id=alias.id, website_email=full_address.address + ) return contact @@ -336,7 +336,9 @@ def replace_header_when_forward(msg: Message, alias: Alias, header: str): try: # NOT allow unicode for contact address - validate_email(contact_email, check_deliverability=False, allow_smtputf8=False) + validate_email( + contact_email, check_deliverability=False, allow_smtputf8=False + ) except EmailNotValidError: LOG.w("invalid contact email %s. %s. Skip", contact_email, headers) continue @@ -572,13 +574,13 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str # handle_email_sent_to_ourself(alias, mb, msg, user) # return [(True, "250 Message accepted for delivery")] - from_header = str(msg["From"]) + from_header = get_header_unicode(msg["From"]) LOG.d("Create or get contact for from_header:%s", from_header) contact = get_or_create_contact(from_header, envelope.mail_from, alias) reply_to_contact = None if msg["Reply-To"]: - reply_to = str(msg["Reply-To"]) + reply_to = get_header_unicode(msg["Reply-To"]) LOG.d("Create or get contact for from_header:%s", reply_to) # ignore when reply-to = alias if reply_to == alias.email: @@ -1299,7 +1301,8 @@ def handle_hotmail_complaint(msg: Message) -> bool: LOG.e("cannot find the alias") return False - _, alias_address = parseaddr_unicode(to_header) + full_address: EmailAddress = address.parse(get_header_unicode(to_header)) + alias_address = full_address.address alias = Alias.get_by(email=alias_address) if not alias: From 638e8137ec3a078f31b789109360e2d5e07ae6ef Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 17:10:05 +0200 Subject: [PATCH 057/144] fix test --- app/api/views/alias.py | 7 +++---- email_handler.py | 4 ++-- tests/api/test_alias.py | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/api/views/alias.py b/app/api/views/alias.py index b1f78b78..64151bb7 100644 --- a/app/api/views/alias.py +++ b/app/api/views/alias.py @@ -18,7 +18,6 @@ from app.api.serializer import ( ) from app.dashboard.views.alias_log import get_alias_log from app.email_utils import ( - is_valid_email, generate_reply_email, ) from app.extensions import db @@ -402,10 +401,10 @@ def create_contact_route(alias_id): return jsonify(error="Contact cannot be empty"), 400 full_address: EmailAddress = address.parse(contact_addr) - contact_name, contact_email = full_address.display_name, full_address.address + if not full_address: + return jsonify(error=f"invalid contact email {contact_addr}"), 400 - if not is_valid_email(contact_email): - return jsonify(error=f"invalid contact email {contact_email}"), 400 + contact_name, contact_email = full_address.display_name, full_address.address contact_email = sanitize_email(contact_email) diff --git a/email_handler.py b/email_handler.py index bcfceab7..03b4946f 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1100,8 +1100,8 @@ def get_mailbox_from_mail_from(mail_from: str, alias) -> Optional[Mailbox]: if mailbox.email == mail_from: return mailbox - for address in mailbox.authorized_addresses: - if address.email == mail_from: + for addr in mailbox.authorized_addresses: + if addr.email == mail_from: LOG.d( "Found an authorized address for %s %s %s", alias, mailbox, address ) diff --git a/tests/api/test_alias.py b/tests/api/test_alias.py index 4b3a7d6a..473cc41e 100644 --- a/tests/api/test_alias.py +++ b/tests/api/test_alias.py @@ -577,11 +577,11 @@ def test_create_contact_route_invalid_contact_email(flask_client): r = flask_client.post( url_for("api.create_contact_route", alias_id=alias.id), - json={"contact": "with space@gmail.com"}, + json={"contact": "@gmail.com"}, ) assert r.status_code == 400 - assert r.json["error"] == "invalid contact email with space@gmail.com" + assert r.json["error"] == "invalid contact email @gmail.com" def test_delete_contact(flask_client): From 41478a57157400be95c13b786fc22193fbe0a827 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 17:26:14 +0200 Subject: [PATCH 058/144] replace parseaddr_unicode by parse_full_address --- app/email_utils.py | 60 +++++++++++---------------------------- tests/test_email_utils.py | 20 +++++-------- tests/test_models.py | 6 ++-- 3 files changed, 27 insertions(+), 59 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index a4e8af5c..d0fd4073 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -7,12 +7,11 @@ import random import re import time from copy import deepcopy -from email.errors import HeaderParseError from email.header import decode_header, Header from email.message import Message from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import make_msgid, formatdate, parseaddr +from email.utils import make_msgid, formatdate from smtplib import SMTP, SMTPServerDisconnected from typing import Tuple, List, Optional, Union @@ -20,6 +19,8 @@ import arrow import dkim import spf from email_validator import validate_email, EmailNotValidError +from flanker.addresslib import address +from flanker.addresslib.address import EmailAddress from jinja2 import Environment, FileSystemLoader from sqlalchemy import func @@ -681,47 +682,6 @@ def get_header_unicode(header: Union[str, Header]) -> str: return ret -def parseaddr_unicode(addr) -> (str, str): - """Like parseaddr() but return name in unicode instead of in RFC 2047 format - Should be used instead of parseaddr() - '=?UTF-8?B?TmjGoW4gTmd1eeG7hW4=?= ' -> ('Nhơn Nguyễn', "abcd@gmail.com") - """ - # sometimes linebreaks are present in addr - addr = addr.replace("\n", "").strip() - name, email = parseaddr(addr) - # email can have whitespace so we can't remove whitespace here - email = email.strip().lower() - if name: - name = name.strip() - try: - decoded_string, charset = decode_header(name)[0] - except HeaderParseError: # fail in case - LOG.w("Can't decode name %s", name) - else: - if charset is not None: - try: - name = decoded_string.decode(charset) - except UnicodeDecodeError: - LOG.w("Cannot decode addr name %s", name) - name = "" - except LookupError: # charset is unknown - LOG.w( - "Cannot decode %s with %s, use utf-8", decoded_string, charset - ) - try: - name = decoded_string.decode("utf-8") - except UnicodeDecodeError: - LOG.w("utf-8 not work on %s", decoded_string) - name = "" - - else: - name = decoded_string - - if type(name) == bytes: - name = name.decode() - return name, email - - def copy(msg: Message) -> Message: """return a copy of message""" try: @@ -1257,3 +1217,17 @@ def should_ignore_bounce(mail_from: str) -> bool: return True return False + + +def parse_full_address(full_address) -> (str, str): + """ + parse the email address full format and return the display name and address + For ex: ab -> (ab, cd@xy.com) + '=?UTF-8?B?TmjGoW4gTmd1eeG7hW4=?= ' -> ('Nhơn Nguyễn', "abcd@gmail.com") + + If the parsing fails, raise ValueError + """ + full_address: EmailAddress = address.parse(full_address) + if full_address is None: + raise ValueError + return full_address.display_name, full_address.address diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index 5e1e7528..3395bcca 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -10,7 +10,6 @@ from app.email_utils import ( email_can_be_used_as_mailbox, delete_header, add_or_replace_header, - parseaddr_unicode, send_email_with_rate_control, copy, get_spam_from_header, @@ -30,6 +29,7 @@ from app.email_utils import ( get_queue_id, should_ignore_bounce, get_header_unicode, + parse_full_address, ) from app.extensions import db from app.models import User, CustomDomain, Alias, Contact, EmailLog, IgnoreBounceSender @@ -100,43 +100,37 @@ def test_add_or_replace_header(): assert msg._headers == [("H", "new")] -def test_parseaddr_unicode(): +def test_parse_full_address(): # only email - assert parseaddr_unicode("abcd@gmail.com") == ( + assert parse_full_address("abcd@gmail.com") == ( "", "abcd@gmail.com", ) # ascii address - assert parseaddr_unicode("First Last ") == ( + assert parse_full_address("First Last ") == ( "First Last", "abcd@gmail.com", ) # Handle quote - assert parseaddr_unicode('"First Last" ') == ( + assert parse_full_address('"First Last" ') == ( "First Last", "abcd@gmail.com", ) # UTF-8 charset - assert parseaddr_unicode("=?UTF-8?B?TmjGoW4gTmd1eeG7hW4=?= ") == ( + assert parse_full_address("=?UTF-8?B?TmjGoW4gTmd1eeG7hW4=?= ") == ( "Nhơn Nguyễn", "abcd@gmail.com", ) # iso-8859-1 charset - assert parseaddr_unicode("=?iso-8859-1?q?p=F6stal?= ") == ( + assert parse_full_address("=?iso-8859-1?q?p=F6stal?= ") == ( "pöstal", "abcd@gmail.com", ) - # when a name can't be decoded, return an empty string - assert parseaddr_unicode("=?UTF-8?B?Cec���?= ") == ( - "", - "test@example.com", - ) - def test_send_email_with_rate_control(flask_client): user = User.create( diff --git a/tests/test_models.py b/tests/test_models.py index 76e621ce..48aa0af5 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -3,7 +3,7 @@ from uuid import UUID import pytest from app.config import EMAIL_DOMAIN, MAX_NB_EMAIL_FREE_PLAN -from app.email_utils import parseaddr_unicode +from app.email_utils import parse_full_address from app.extensions import db from app.models import ( generate_email, @@ -159,8 +159,8 @@ def test_new_addr(flask_client): == "=?utf-8?q?Nh=C6=A1n_Nguy=E1=BB=85n_-_abcd_at_example=2Ecom?= " ) - # sanity check for parseaddr_unicode - assert parseaddr_unicode(c1.new_addr()) == ( + # sanity check + assert parse_full_address(c1.new_addr()) == ( "Nhơn Nguyễn - abcd at example.com", "rep@sl", ) From 61d9f7ee437037a0c4323f02e4c9704e33dff858 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 17:31:29 +0200 Subject: [PATCH 059/144] refactor --- app/dashboard/views/alias_contact_manager.py | 8 ++--- email_handler.py | 36 +++++++++++--------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/app/dashboard/views/alias_contact_manager.py b/app/dashboard/views/alias_contact_manager.py index 6d80f5b5..535fbf2a 100644 --- a/app/dashboard/views/alias_contact_manager.py +++ b/app/dashboard/views/alias_contact_manager.py @@ -15,6 +15,7 @@ from app.dashboard.base import dashboard_bp from app.email_utils import ( is_valid_email, generate_reply_email, + parse_full_address, ) from app.extensions import db from app.log import LOG @@ -183,12 +184,7 @@ def alias_contact_manager(alias_id): contact_addr = new_contact_form.email.data.strip() try: - full_address: EmailAddress = address.parse(contact_addr) - contact_name, contact_email = ( - full_address.display_name, - full_address.address, - ) - contact_email = sanitize_email(contact_email) + contact_name, contact_email = parse_full_address(contact_addr) except Exception: flash(f"{contact_addr} is invalid", "error") return redirect( diff --git a/email_handler.py b/email_handler.py index 03b4946f..c4ce638f 100644 --- a/email_handler.py +++ b/email_handler.py @@ -117,6 +117,7 @@ from app.email_utils import ( get_queue_id, should_ignore_bounce, get_orig_message_from_outlook_complaint, + parse_full_address, ) from app.extensions import db from app.log import LOG, set_message_id @@ -179,8 +180,11 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con """ contact_from_header is the RFC 2047 format FROM header """ - full_address: EmailAddress = address.parse(from_header) - contact_name, contact_email = full_address.display_name, full_address.address + try: + contact_name, contact_email = parse_full_address(from_header) + except ValueError: + contact_name, contact_email = "", "" + if not is_valid_email(contact_email): # From header is wrongly formatted, try with mail_from if mail_from and mail_from != "<>": @@ -273,23 +277,26 @@ def get_or_create_reply_to_contact( """ Get or create the contact for the Reply-To header """ - full_address: EmailAddress = address.parse(reply_to_header) + try: + contact_name, contact_address = parse_full_address(reply_to_header) + except ValueError: + return - if not is_valid_email(full_address.address): + if not is_valid_email(contact_address): LOG.w( "invalid reply-to address %s. Parse from %s", - full_address, + contact_address, reply_to_header, ) return None - contact = Contact.get_by(alias_id=alias.id, website_email=full_address.address) + contact = Contact.get_by(alias_id=alias.id, website_email=contact_address) if contact: return contact else: LOG.d( "create contact %s for alias %s via reply-to header", - full_address.address, + contact_address, alias, reply_to_header, ) @@ -298,17 +305,15 @@ def get_or_create_reply_to_contact( contact = Contact.create( user_id=alias.user_id, alias_id=alias.id, - website_email=full_address.address, - name=full_address.display_name, - reply_email=generate_reply_email(full_address.address, alias.user), + website_email=contact_address, + name=contact_name, + reply_email=generate_reply_email(contact_address, alias.user), ) db.session.commit() except IntegrityError: - LOG.w("Contact %s %s already exist", alias, full_address.address) + LOG.w("Contact %s %s already exist", alias, contact_address) db.session.rollback() - contact = Contact.get_by( - alias_id=alias.id, website_email=full_address.address - ) + contact = Contact.get_by(alias_id=alias.id, website_email=contact_address) return contact @@ -1301,8 +1306,7 @@ def handle_hotmail_complaint(msg: Message) -> bool: LOG.e("cannot find the alias") return False - full_address: EmailAddress = address.parse(get_header_unicode(to_header)) - alias_address = full_address.address + _, alias_address = parse_full_address(get_header_unicode(to_header)) alias = Alias.get_by(email=alias_address) if not alias: From e5a388dffb9419c0f6493ede763321e68c7cf17c Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 17:37:33 +0200 Subject: [PATCH 060/144] optimize import --- app/dashboard/views/alias_contact_manager.py | 3 --- app/paddle_utils.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/app/dashboard/views/alias_contact_manager.py b/app/dashboard/views/alias_contact_manager.py index 535fbf2a..c9e644b1 100644 --- a/app/dashboard/views/alias_contact_manager.py +++ b/app/dashboard/views/alias_contact_manager.py @@ -1,8 +1,6 @@ from dataclasses import dataclass from operator import or_ -from flanker.addresslib import address -from flanker.addresslib.address import EmailAddress from flask import render_template, request, redirect, flash from flask import url_for from flask_login import login_required, current_user @@ -20,7 +18,6 @@ from app.email_utils import ( from app.extensions import db from app.log import LOG from app.models import Alias, Contact, EmailLog -from app.utils import sanitize_email def email_validator(): diff --git a/app/paddle_utils.py b/app/paddle_utils.py index b2668bc1..3ee702d4 100644 --- a/app/paddle_utils.py +++ b/app/paddle_utils.py @@ -10,13 +10,11 @@ import collections import phpserialize import requests from Crypto.Hash import SHA1 - # Crypto can be found at https://pypi.org/project/pycryptodome/ from Crypto.PublicKey import RSA from Crypto.Signature import PKCS1_v1_5 from app.config import PADDLE_PUBLIC_KEY_PATH, PADDLE_VENDOR_ID, PADDLE_AUTH_CODE - # Your Paddle public key. from app.log import LOG from app.models import User From 4949afc791c37fc9dfa7b1277b3721a4246296d2 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 17:42:07 +0200 Subject: [PATCH 061/144] black --- app/paddle_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/paddle_utils.py b/app/paddle_utils.py index 3ee702d4..b2668bc1 100644 --- a/app/paddle_utils.py +++ b/app/paddle_utils.py @@ -10,11 +10,13 @@ import collections import phpserialize import requests from Crypto.Hash import SHA1 + # Crypto can be found at https://pypi.org/project/pycryptodome/ from Crypto.PublicKey import RSA from Crypto.Signature import PKCS1_v1_5 from app.config import PADDLE_PUBLIC_KEY_PATH, PADDLE_VENDOR_ID, PADDLE_AUTH_CODE + # Your Paddle public key. from app.log import LOG from app.models import User From f53e8c1af87369a4e942fcc27770676c1c115aa2 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 17:48:36 +0200 Subject: [PATCH 062/144] refactor --- email_handler.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/email_handler.py b/email_handler.py index c4ce638f..e7f1dcbb 100644 --- a/email_handler.py +++ b/email_handler.py @@ -549,14 +549,17 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str is_success indicates whether an email has been delivered and smtp_status is the SMTP Status ("250 Message accepted", "550 Non-existent email address", etc) """ - address = rcpt_to # alias@SL + alias_address = rcpt_to # alias@SL - alias = Alias.get_by(email=address) + alias = Alias.get_by(email=alias_address) if not alias: - LOG.d("alias %s not exist. Try to see if it can be created on the fly", address) - alias = try_auto_create(address) + LOG.d( + "alias %s not exist. Try to see if it can be created on the fly", + alias_address, + ) + alias = try_auto_create(alias_address) if not alias: - LOG.d("alias %s cannot be created on-the-fly, return 550", address) + LOG.d("alias %s cannot be created on-the-fly, return 550", alias_address) if should_ignore_bounce(envelope.mail_from): return [(True, status.E207)] else: From 62044e6db13fecc8d99329d08a8161751d12d179 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 18:14:51 +0200 Subject: [PATCH 063/144] Add User.ignore_loop_email column --- app/models.py | 6 ++++ .../versions/2021_091018_4d3f91ddf3e9_.py | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 migrations/versions/2021_091018_4d3f91ddf3e9_.py diff --git a/app/models.py b/app/models.py index 6997b687..de1972b4 100644 --- a/app/models.py +++ b/app/models.py @@ -336,6 +336,12 @@ class User(db.Model, ModelMixin, UserMixin, PasswordOracle): db.Boolean, default=False, nullable=False, server_default="0" ) + # ignore emails send from a mailbox to its alias. This can happen when replying all to a forwarded email + # can automatically re-includes the alias + ignore_loop_email = db.Column( + db.Boolean, default=False, nullable=False, server_default="0" + ) + @classmethod def create(cls, email, name="", password=None, **kwargs): user: User = super(User, cls).create(email=email, name=name, **kwargs) diff --git a/migrations/versions/2021_091018_4d3f91ddf3e9_.py b/migrations/versions/2021_091018_4d3f91ddf3e9_.py new file mode 100644 index 00000000..2b2176a9 --- /dev/null +++ b/migrations/versions/2021_091018_4d3f91ddf3e9_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: 4d3f91ddf3e9 +Revises: 916a5257d18c +Create Date: 2021-09-10 18:12:21.374836 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '4d3f91ddf3e9' +down_revision = '916a5257d18c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('users', sa.Column('ignore_loop_email', sa.Boolean(), server_default='0', nullable=False)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('users', 'ignore_loop_email') + # ### end Alembic commands ### From 6e42e536dbabda5dffcd9a523897420434bf5731 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 10 Sep 2021 18:15:22 +0200 Subject: [PATCH 064/144] ignore email sent from a mailbox to its alias if user.ignore_loop_email --- app/email/status.py | 2 ++ email_handler.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/email/status.py b/app/email/status.py index 28e95ae7..5e2b7b59 100644 --- a/app/email/status.py +++ b/app/email/status.py @@ -13,6 +13,8 @@ E207 = "250 SL E207 No bounce report" E208 = "250 SL E208 Hotmail complaint handled" +E209 = "250 SL E209 Email Loop" + # 4** errors # E401 = "421 SL E401 Retry later" E402 = "421 SL E402 Encryption failed - Retry later" diff --git a/email_handler.py b/email_handler.py index e7f1dcbb..fa4bd926 100644 --- a/email_handler.py +++ b/email_handler.py @@ -574,13 +574,14 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str else: return [(False, status.E504)] - # mail_from = envelope.mail_from - # for mb in alias.mailboxes: - # # email send from a mailbox to alias - # if mb.email == mail_from: - # LOG.w("cycle email sent from %s to %s", mb, alias) - # handle_email_sent_to_ourself(alias, mb, msg, user) - # return [(True, "250 Message accepted for delivery")] + if user.ignore_loop_email: + mail_from = envelope.mail_from + for mb in alias.mailboxes: + # email send from a mailbox to its alias + if mb.email == mail_from: + LOG.w("cycle email sent from %s to %s", mb, alias) + handle_email_sent_to_ourself(alias, mb, msg, user) + return [(True, status.E209)] from_header = get_header_unicode(msg["From"]) LOG.d("Create or get contact for from_header:%s", from_header) From b3012376c3d1efd9fc67111a71ff09841c7a67cf Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 13 Sep 2021 16:04:32 +0200 Subject: [PATCH 065/144] use another DKIM header if one fails --- app/config.py | 1 - app/email_utils.py | 27 +++++++++++++++++++++++++-- email_handler.py | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/app/config.py b/app/config.py index a7a7073f..b3d132f6 100644 --- a/app/config.py +++ b/app/config.py @@ -157,7 +157,6 @@ DISABLE_ALIAS_SUFFIX = "DISABLE_ALIAS_SUFFIX" in os.environ UNSUBSCRIBER = os.environ.get("UNSUBSCRIBER") DKIM_SELECTOR = b"dkim" -DKIM_HEADERS = [b"from", b"to"] DKIM_PRIVATE_KEY = None if "DKIM_PRIVATE_KEY_PATH" in os.environ: diff --git a/app/email_utils.py b/app/email_utils.py index d0fd4073..c6a5659a 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -31,7 +31,6 @@ from app.config import ( NOT_SEND_EMAIL, DKIM_SELECTOR, DKIM_PRIVATE_KEY, - DKIM_HEADERS, ALIAS_DOMAINS, SUPPORT_NAME, POSTFIX_SUBMISSION_TLS, @@ -391,7 +390,31 @@ def get_email_domain_part(address): return address[address.find("@") + 1 :] +# headers used to DKIM sign in order of preference +_DKIM_HEADERS = [ + [b"Message-ID", b"Date", b"subject", b"from", b"to"], + [b"from", b"to"], + [b"Message-ID", b"Date"], + [b"from"], +] + + def add_dkim_signature(msg: Message, email_domain: str): + for dkim_headers in _DKIM_HEADERS: + try: + add_dkim_signature_with_header(msg, email_domain, dkim_headers) + return + except dkim.DKIMException: + LOG.w("DKIM fail with %s", dkim_headers) + # try with another headers + continue + + raise Exception("Cannot create DKIM signature") + + +def add_dkim_signature_with_header( + msg: Message, email_domain: str, dkim_headers: [bytes] +): delete_header(msg, "DKIM-Signature") # Specify headers in "byte" form @@ -402,7 +425,7 @@ def add_dkim_signature(msg: Message, email_domain: str): DKIM_SELECTOR, email_domain.encode(), DKIM_PRIVATE_KEY.encode(), - include_headers=DKIM_HEADERS, + include_headers=dkim_headers, ) sig = sig.decode() diff --git a/email_handler.py b/email_handler.py index fa4bd926..b6aa5dde 100644 --- a/email_handler.py +++ b/email_handler.py @@ -577,7 +577,7 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str if user.ignore_loop_email: mail_from = envelope.mail_from for mb in alias.mailboxes: - # email send from a mailbox to its alias + # email sent from a mailbox to its alias if mb.email == mail_from: LOG.w("cycle email sent from %s to %s", mb, alias) handle_email_sent_to_ourself(alias, mb, msg, user) From 106358da5f2c5f124e5240b01bfe3c753d339aa3 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 13 Sep 2021 19:49:40 +0200 Subject: [PATCH 066/144] handle yahoo complaint --- app/config.py | 1 + app/email/status.py | 2 + app/email_utils.py | 13 ++++- email_handler.py | 55 ++++++++++++++++++- .../emails/transactional/yahoo-complaint.html | 33 +++++++++++ .../transactional/yahoo-complaint.txt.jinja2 | 17 ++++++ 6 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 templates/emails/transactional/yahoo-complaint.html create mode 100644 templates/emails/transactional/yahoo-complaint.txt.jinja2 diff --git a/app/config.py b/app/config.py index b3d132f6..ecffc2e3 100644 --- a/app/config.py +++ b/app/config.py @@ -333,6 +333,7 @@ AlERT_WRONG_MX_RECORD_CUSTOM_DOMAIN = "custom_domain_mx_record_issue" ALERT_DIRECTORY_DISABLED_ALIAS_CREATION = "alert_directory_disabled_alias_creation" ALERT_HOTMAIL_COMPLAINT = "alert_hotmail_complaint" +ALERT_YAHOO_COMPLAINT = "alert_yahoo_complaint" # <<<<< END ALERT EMAIL >>>> diff --git a/app/email/status.py b/app/email/status.py index 5e2b7b59..b2fbfef9 100644 --- a/app/email/status.py +++ b/app/email/status.py @@ -15,6 +15,8 @@ E208 = "250 SL E208 Hotmail complaint handled" E209 = "250 SL E209 Email Loop" +E210 = "250 SL E210 Yahoo complaint handled" + # 4** errors # E401 = "421 SL E401 Retry later" E402 = "421 SL E402 Encryption failed - Retry later" diff --git a/app/email_utils.py b/app/email_utils.py index c6a5659a..97aa3555 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -592,7 +592,6 @@ def get_orig_message_from_bounce(msg: Message) -> Message: def get_orig_message_from_outlook_complaint(msg: Message) -> Message: - """parse the original email from Bounce""" i = 0 for part in msg.walk(): i += 1 @@ -604,6 +603,18 @@ def get_orig_message_from_outlook_complaint(msg: Message) -> Message: return part +def get_orig_message_from_yahoo_complaint(msg: Message) -> Message: + i = 0 + for part in msg.walk(): + i += 1 + + # 1st part is the container + # 2nd part is the empty body + # 6th is original message + if i == 6: + return part + + def get_header_from_bounce(msg: Message, header: str) -> str: """using regex to get header value from bounce message get_orig_message_from_bounce is better. This should be the last option diff --git a/email_handler.py b/email_handler.py index b6aa5dde..d0f6a74c 100644 --- a/email_handler.py +++ b/email_handler.py @@ -81,6 +81,7 @@ from app.config import ( NEWRELIC_CONFIG_PATH, POSTMASTER, ALERT_HOTMAIL_COMPLAINT, + ALERT_YAHOO_COMPLAINT, ) from app.email import status from app.email.rate_limit import rate_limited @@ -118,6 +119,7 @@ from app.email_utils import ( should_ignore_bounce, get_orig_message_from_outlook_complaint, parse_full_address, + get_orig_message_from_yahoo_complaint, ) from app.extensions import db from app.log import LOG, set_message_id @@ -1314,7 +1316,7 @@ def handle_hotmail_complaint(msg: Message) -> bool: alias = Alias.get_by(email=alias_address) if not alias: - LOG.d("No alias for %s", alias_address) + LOG.w("No alias for %s", alias_address) return False user = alias.user @@ -1339,6 +1341,46 @@ def handle_hotmail_complaint(msg: Message) -> bool: return True +def handle_yahoo_complaint(msg: Message) -> bool: + """ + Handle yahoo complaint sent to postmaster + Return True if the complaint can be handled, False otherwise + """ + orig_msg = get_orig_message_from_yahoo_complaint(msg) + to_header = orig_msg["To"] + if not to_header: + LOG.e("cannot find the alias") + return False + + _, alias_address = parse_full_address(get_header_unicode(to_header)) + alias = Alias.get_by(email=alias_address) + + if not alias: + LOG.w("No alias for %s", alias_address) + return False + + user = alias.user + LOG.w("Handle yahoo complaint for %s %s %s", alias, user, alias.mailboxes) + + send_email_with_rate_control( + user, + ALERT_YAHOO_COMPLAINT, + user.email, + f"Yahoo abuse report", + render( + "transactional/yahoo-complaint.txt.jinja2", + alias=alias, + ), + render( + "transactional/yahoo-complaint.html", + alias=alias, + ), + max_nb_alert=2, + ) + + return True + + def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog): """ Handle reply phase bounce @@ -1758,6 +1800,17 @@ def handle(envelope: Envelope) -> str: if handle_hotmail_complaint(msg): return status.E208 + if ( + len(rcpt_tos) == 1 + and mail_from == "feedback@arf.mail.yahoo.com" + and rcpt_tos[0] == POSTMASTER + ): + LOG.w("Handle yahoo complaint") + + # if the complaint cannot be handled, forward it normally + if handle_yahoo_complaint(msg): + return status.E210 + # Handle bounce if ( len(rcpt_tos) == 1 diff --git a/templates/emails/transactional/yahoo-complaint.html b/templates/emails/transactional/yahoo-complaint.html new file mode 100644 index 00000000..62c69d6e --- /dev/null +++ b/templates/emails/transactional/yahoo-complaint.html @@ -0,0 +1,33 @@ +{% extends "base.html" %} + +{% block content %} + {% call text() %} + This is SimpleLogin team.
+ Yahoo has informed us about an email sent to {{ alias.email }} that might have been marked as spam. + {% endcall %} + + {% call text() %} + Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and + is a violation of our terms and condition. + {% endcall %} + + {% call text() %} + If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. + {% endcall %} + + {% call text() %} + If SimpleLogin isn’t useful for you, please know that you can simply delete your account on the Settings page. + {% endcall %} + + {% call text() %} + Looking to hear back from you. + {% endcall %} + + {% call text() %} + Best,
+ SimpleLogin Team. + {% endcall %} + +{% endblock %} + + diff --git a/templates/emails/transactional/yahoo-complaint.txt.jinja2 b/templates/emails/transactional/yahoo-complaint.txt.jinja2 new file mode 100644 index 00000000..ad1d6c59 --- /dev/null +++ b/templates/emails/transactional/yahoo-complaint.txt.jinja2 @@ -0,0 +1,17 @@ +Hi, + +This is SimpleLogin team. + +Yahoo has informed us about an email sent to {{ alias.email }} that might have been marked as spam. + +Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and + is a violation of our terms and condition. + +If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. + +If SimpleLogin isn’t useful for you, please know that you can simply delete your account on the Settings page. + +Looking to hear back from you. + +Best, +SimpleLogin Team. \ No newline at end of file From e4ed192cce8fd6b30f96c27f4f5af3ff4ba16639 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 13 Sep 2021 19:50:15 +0200 Subject: [PATCH 067/144] rename --- app/email_utils.py | 2 +- email_handler.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 97aa3555..6c387840 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -591,7 +591,7 @@ def get_orig_message_from_bounce(msg: Message) -> Message: return part -def get_orig_message_from_outlook_complaint(msg: Message) -> Message: +def get_orig_message_from_hotmail_complaint(msg: Message) -> Message: i = 0 for part in msg.walk(): i += 1 diff --git a/email_handler.py b/email_handler.py index d0f6a74c..342f39e0 100644 --- a/email_handler.py +++ b/email_handler.py @@ -117,7 +117,7 @@ from app.email_utils import ( sanitize_header, get_queue_id, should_ignore_bounce, - get_orig_message_from_outlook_complaint, + get_orig_message_from_hotmail_complaint, parse_full_address, get_orig_message_from_yahoo_complaint, ) @@ -1306,7 +1306,7 @@ def handle_hotmail_complaint(msg: Message) -> bool: Handle hotmail complaint sent to postmaster Return True if the complaint can be handled, False otherwise """ - orig_msg = get_orig_message_from_outlook_complaint(msg) + orig_msg = get_orig_message_from_hotmail_complaint(msg) to_header = orig_msg["To"] if not to_header: LOG.e("cannot find the alias") From 8990895dd267d2e096b0d5864a15508642295325 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 14 Sep 2021 09:03:25 +0200 Subject: [PATCH 068/144] improve email wording --- templates/emails/transactional/hotmail-complaint.html | 11 ++++++++--- .../emails/transactional/hotmail-complaint.txt.jinja2 | 10 ++++++---- templates/emails/transactional/yahoo-complaint.html | 3 ++- .../emails/transactional/yahoo-complaint.txt.jinja2 | 4 ++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/templates/emails/transactional/hotmail-complaint.html b/templates/emails/transactional/hotmail-complaint.html index 6c67c44e..bec8c7ad 100644 --- a/templates/emails/transactional/hotmail-complaint.html +++ b/templates/emails/transactional/hotmail-complaint.html @@ -3,11 +3,13 @@ {% block content %} {% call text() %} This is SimpleLogin team.
- Hotmail has informed us about an email sent to {{ alias.email }} that might have been marked as spam. + Hotmail has informed us about an email sent to {{ alias.email }} that might have been considered as spam, + either by you or by Hotmail spam filter. {% endcall %} {% call text() %} - Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and + Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, + has a negative effect for all users and is a violation of our terms and condition. {% endcall %} @@ -16,7 +18,10 @@ {% endcall %} {% call text() %} - If SimpleLogin isn’t useful for you, please know that you can simply delete your account on the Settings page. + If somehow Hotmail considers a forwarded email as Spam, it will help us if you can move the email out of the Spam + folder. + You can also set up a filter to avoid this from happening in the future using this guide at + https://simplelogin.io/help/ {% endcall %} {% call text() %} diff --git a/templates/emails/transactional/hotmail-complaint.txt.jinja2 b/templates/emails/transactional/hotmail-complaint.txt.jinja2 index 1e8d611e..b74a933d 100644 --- a/templates/emails/transactional/hotmail-complaint.txt.jinja2 +++ b/templates/emails/transactional/hotmail-complaint.txt.jinja2 @@ -2,14 +2,16 @@ Hi, This is SimpleLogin team. -Hotmail has informed us about an email sent to {{ alias.email }} that might have been marked as spam. +Hotmail has informed us about an email sent to {{ alias.email }} that might have been considered as spam, +either by you or by Hotmail spam filter. -Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and - is a violation of our terms and condition. +Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, +has a negative effect for all users and is a violation of our terms and condition. If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. -If SimpleLogin isn’t useful for you, please know that you can simply delete your account on the Settings page. +If somehow Hotmail considers a forwarded email as Spam, it will help us if you can move the email out of the Spam folder. +You can also set up a filter to avoid this from happening in the future using this guide at https://simplelogin.io/help/ Looking to hear back from you. diff --git a/templates/emails/transactional/yahoo-complaint.html b/templates/emails/transactional/yahoo-complaint.html index 62c69d6e..d43ca6a1 100644 --- a/templates/emails/transactional/yahoo-complaint.html +++ b/templates/emails/transactional/yahoo-complaint.html @@ -7,7 +7,8 @@ {% endcall %} {% call text() %} - Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and + Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, + has a negative effect for all users and is a violation of our terms and condition. {% endcall %} diff --git a/templates/emails/transactional/yahoo-complaint.txt.jinja2 b/templates/emails/transactional/yahoo-complaint.txt.jinja2 index ad1d6c59..9c2a5127 100644 --- a/templates/emails/transactional/yahoo-complaint.txt.jinja2 +++ b/templates/emails/transactional/yahoo-complaint.txt.jinja2 @@ -4,8 +4,8 @@ This is SimpleLogin team. Yahoo has informed us about an email sent to {{ alias.email }} that might have been marked as spam. -Putting a forwarded email into Spam affects SimpleLogin email delivery, has a negative effect for all users and - is a violation of our terms and condition. +Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, +has a negative effect for all users and is a violation of our terms and condition. If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. From 567bee9a0b33d681ebbe6a13190e902ff5cfa7bb Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 14 Sep 2021 09:13:39 +0200 Subject: [PATCH 069/144] add exception trace when dkim fails --- app/email_utils.py | 8 ++++---- app/log.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 6c387840..9b3140cf 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -392,10 +392,10 @@ def get_email_domain_part(address): # headers used to DKIM sign in order of preference _DKIM_HEADERS = [ - [b"Message-ID", b"Date", b"subject", b"from", b"to"], - [b"from", b"to"], + [b"Message-ID", b"Date", b"Subject", b"From", b"To"], + [b"From", b"To"], [b"Message-ID", b"Date"], - [b"from"], + [b"From"], ] @@ -405,7 +405,7 @@ def add_dkim_signature(msg: Message, email_domain: str): add_dkim_signature_with_header(msg, email_domain, dkim_headers) return except dkim.DKIMException: - LOG.w("DKIM fail with %s", dkim_headers) + LOG.w("DKIM fail with %s", dkim_headers, exc_info=True) # try with another headers continue diff --git a/app/log.py b/app/log.py index 09065457..184162a4 100644 --- a/app/log.py +++ b/app/log.py @@ -42,7 +42,7 @@ def _get_console_handler(): return console_handler -def _get_logger(name): +def _get_logger(name) -> logging.Logger: logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) From 1cc8f7f2e34b3f5193968cee7fe8113db27b59fc Mon Sep 17 00:00:00 2001 From: David Siregar <32016658+Dattito@users.noreply.github.com> Date: Tue, 14 Sep 2021 21:28:51 +0200 Subject: [PATCH 070/144] fixed wrong description of error --- app/api/views/new_random_alias.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/views/new_random_alias.py b/app/api/views/new_random_alias.py index c67b3aae..19d2e3bd 100644 --- a/app/api/views/new_random_alias.py +++ b/app/api/views/new_random_alias.py @@ -48,7 +48,7 @@ def new_random_alias(): elif mode == "uuid": scheme = AliasGeneratorEnum.uuid.value else: - return jsonify(error=f"{mode} must be either word or alias"), 400 + return jsonify(error=f"{mode} must be either word or uuid"), 400 alias = Alias.create_new_random(user=user, scheme=scheme, note=note) db.session.commit() From 3441d2ccf1b75c3e9899b54c30d7497b38964e29 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 15 Sep 2021 09:28:08 +0200 Subject: [PATCH 071/144] add new param TEMP_DIR --- app/config.py | 3 +++ example.env | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index ecffc2e3..54b36a97 100644 --- a/app/config.py +++ b/app/config.py @@ -397,3 +397,6 @@ HIBP_API_KEYS = sl_getenv("HIBP_API_KEYS", list) or [] NEWRELIC_CONFIG_PATH = os.environ.get("NEWRELIC_CONFIG_PATH") POSTMASTER = os.environ.get("POSTMASTER") + +# store temporary files, especially for debugging +TEMP_DIR = os.environ.get("TEMP_DIR") diff --git a/example.env b/example.env index 9dfac231..12279ed1 100644 --- a/example.env +++ b/example.env @@ -171,4 +171,8 @@ DISABLE_ONBOARDING=true # HIBP_API_KEYS=[] # NewRelic Config File Path -# NEWRELIC_CONFIG_PATH = /path/newrelic.ini \ No newline at end of file +# NEWRELIC_CONFIG_PATH = /path/newrelic.ini + +# POSTMASTER = postmaster@example.com + +# TEMP_DIR = /tmp \ No newline at end of file From 144418ae47b7f714e2ad5b0c3daf8dbf33418982 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 15 Sep 2021 09:28:27 +0200 Subject: [PATCH 072/144] use debug level when postfix queue id can't be parsed --- email_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 342f39e0..ae531297 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1738,7 +1738,7 @@ def handle(envelope: Envelope) -> str: if postfix_queue_id: set_message_id(postfix_queue_id) else: - LOG.w("Cannot parse Postfix queue ID from %s", msg["Received"]) + LOG.d("Cannot parse Postfix queue ID from %s", msg["Received"]) if should_ignore(mail_from, rcpt_tos): LOG.w("Ignore email mail_from=%s rcpt_to=%s", mail_from, rcpt_tos) From 695a628e689648802b120e290918e9866755849e Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 15 Sep 2021 09:28:54 +0200 Subject: [PATCH 073/144] save email that can't be DKIM signed to temp dir to investigate --- app/email_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/email_utils.py b/app/email_utils.py index 9b3140cf..ac219f6b 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -6,6 +6,7 @@ import quopri import random import re import time +import uuid from copy import deepcopy from email.header import decode_header, Header from email.message import Message @@ -45,6 +46,7 @@ from app.config import ( TRANSACTIONAL_BOUNCE_EMAIL, ALERT_SPF, POSTFIX_PORT_FORWARD, + TEMP_DIR, ) from app.dns_utils import get_mx_domains from app.extensions import db @@ -409,6 +411,14 @@ def add_dkim_signature(msg: Message, email_domain: str): # try with another headers continue + # To investigate why some emails can't be DKIM signed. todo: remove + if TEMP_DIR: + file_name = str(uuid.uuid4()) + ".eml" + with open(os.path.join(TEMP_DIR, file_name), "wb") as f: + f.write(msg.as_bytes()) + + LOG.w("email saved to %s", file_name) + raise Exception("Cannot create DKIM signature") From 981f6ecfb224f5ed2df7c7347d365123fe5ea7ea Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Thu, 16 Sep 2021 18:02:45 +0200 Subject: [PATCH 074/144] handle the case pg_trgm is already loaded --- migrations/versions/2021_082012_424808e1fe49_.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/migrations/versions/2021_082012_424808e1fe49_.py b/migrations/versions/2021_082012_424808e1fe49_.py index 2038c5e3..ffb61f96 100644 --- a/migrations/versions/2021_082012_424808e1fe49_.py +++ b/migrations/versions/2021_082012_424808e1fe49_.py @@ -8,7 +8,8 @@ Create Date: 2021-08-20 12:11:57.901994 import sqlalchemy_utils from alembic import op import sqlalchemy as sa - +from psycopg2 import errors +from psycopg2.errorcodes import DUPLICATE_OBJECT # revision identifiers, used by Alembic. revision = '424808e1fe49' @@ -19,7 +20,14 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.execute('CREATE EXTENSION pg_trgm') + try: + op.execute('CREATE EXTENSION pg_trgm') + # thanks to https://stackoverflow.com/a/58743364/1428034 ! + except sa.exc.ProgrammingError as e: + if isinstance(e.orig, errors.lookup(DUPLICATE_OBJECT)): + print(">>> pg_trgm already loaded, ignore") + op.execute("Rollback") + op.create_index('note_pg_trgm_index', 'alias', ['note'], unique=False, postgresql_ops={'note': 'gin_trgm_ops'}, postgresql_using='gin') # ### end Alembic commands ### From 68d33ea85b1f2473d077d88048fec63b933812aa Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 10:30:26 +0200 Subject: [PATCH 075/144] upgrade flask-sqlalchemy to 2.5.1 --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 199d9e32..8f9f44d3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -589,7 +589,7 @@ simplejson = "*" [[package]] name = "flask-sqlalchemy" -version = "2.4.4" +version = "2.5.1" description = "Adds SQLAlchemy support to your Flask application." category = "main" optional = false @@ -2086,8 +2086,8 @@ flask-profiler = [ {file = "flask_profiler-1.8.1.tar.gz", hash = "sha256:fc9f2875a4f22223ddc04ffacd75792854162c4cdbef165598a51f898521ac51"}, ] flask-sqlalchemy = [ - {file = "Flask-SQLAlchemy-2.4.4.tar.gz", hash = "sha256:bfc7150eaf809b1c283879302f04c42791136060c6eeb12c0c6674fb1291fae5"}, - {file = "Flask_SQLAlchemy-2.4.4-py2.py3-none-any.whl", hash = "sha256:05b31d2034dd3f2a685cbbae4cfc4ed906b2a733cff7964ada450fd5e462b84e"}, + {file = "Flask-SQLAlchemy-2.5.1.tar.gz", hash = "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912"}, + {file = "Flask_SQLAlchemy-2.5.1-py2.py3-none-any.whl", hash = "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390"}, ] flask-wtf = [ {file = "Flask-WTF-0.14.3.tar.gz", hash = "sha256:d417e3a0008b5ba583da1763e4db0f55a1269d9dd91dcc3eb3c026d3c5dbd720"}, From 785a6193857f3381260c10bab94820f99b8efea6 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Thu, 16 Sep 2021 09:12:53 +0200 Subject: [PATCH 076/144] Add debug_info decorator --- app/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/utils.py b/app/utils.py index 10e3ab67..d90acfdf 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,6 +1,8 @@ import random import string +import time import urllib.parse +from functools import wraps from unidecode import unidecode @@ -73,3 +75,15 @@ def sanitize_email(email_address: str) -> str: def query2str(query): """Useful utility method to print out a SQLAlchemy query""" return query.statement.compile(compile_kwargs={"literal_binds": True}) + + +def debug_info(func): + @wraps(func) + def wrap(*args, **kwargs): + start = time.time() + LOG.d("start %s %s %s", func.__name__, args, kwargs) + ret = func(*args, **kwargs) + LOG.d("finish %s. Takes %s seconds", func.__name__, time.time() - start) + return ret + + return wrap From 512ade83b49854f48d09a07dab391b005b7e04a6 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 11:38:07 +0200 Subject: [PATCH 077/144] improve wording & styling on custom domain info page --- .../dashboard/domain_detail/base.html | 2 +- .../dashboard/domain_detail/info.html | 54 +++++++++---------- static/style.css | 6 ++- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/base.html b/app/dashboard/templates/dashboard/domain_detail/base.html index a2cd6cc7..4e9112bd 100644 --- a/app/dashboard/templates/dashboard/domain_detail/base.html +++ b/app/dashboard/templates/dashboard/domain_detail/base.html @@ -26,7 +26,7 @@
-
+
{% block domain_detail_content %} {% endblock %}
diff --git a/app/dashboard/templates/dashboard/domain_detail/info.html b/app/dashboard/templates/dashboard/domain_detail/info.html index cc3b7a57..ba70d6db 100644 --- a/app/dashboard/templates/dashboard/domain_detail/info.html +++ b/app/dashboard/templates/dashboard/domain_detail/info.html @@ -7,24 +7,23 @@ {% endblock %} {% block domain_detail_content %} -

{{ custom_domain.domain }}

+

{{ custom_domain.domain }}

-
Created {{ custom_domain.created_at | dt }}
- - {{ nb_alias }} aliases +
Created {{ custom_domain.created_at | dt }}. {{ nb_alias }} aliases
-
Catch All
-
- This feature allows you to create aliases on the fly. +

Catch All

+
+ Create aliases on the fly. + Simply use anything@{{ custom_domain.domain }} - next time you need an email address.
- The alias will be created the first time it receives an email - and automatically belong to {{ custom_domain.domain }} mailboxes ( + next time you need an alias: it'll be automatically + created the first time it receives an email.
+ The new alias will belong to {% for mailbox in custom_domain.mailboxes %} {{ mailbox.email }} {% if not loop.last %},{% endif %} - {% endfor %}) + {% endfor %}
@@ -47,33 +46,32 @@
-
Default Alias Name
-
- This name will be used as the default alias name when you send - or reply from an alias, unless overwritten by the alias specific name. +

Default Display Name

+
+ Default display name for aliases created with {{ custom_domain.domain }} + unless overwritten by the alias display name.
- +
- + placeholder="Alias Display Name">
- + {% if custom_domain.name %} - + {% endif %}
-
Random Prefix Generation
-
- A random prefix can be generated for this domain for usage in the New Alias - feature. +

Random Prefix Generation

+
+ Add a random prefix for this domain when creating a new alias.
@@ -96,14 +94,14 @@
-

Delete Domain

-
Please note that this operation is irreversible. - All aliases associated with this domain will be also deleted. +

Delete Domain

+
This operation is irreversible. + All aliases associated with this domain will be deleted.
- Delete domain + Delete domain
{% endblock %} diff --git a/static/style.css b/static/style.css index d8f0c233..18ba81da 100644 --- a/static/style.css +++ b/static/style.css @@ -165,4 +165,8 @@ textarea.parsley-error { .parsley-errors-list.filled { opacity: 1; } -/* END Parsley CSS */ \ No newline at end of file +/* END Parsley CSS */ + +.domain_detail_content { + font-size: 15px; +} \ No newline at end of file From b990c052ac0fbe173e01a6c0cdeaa08ac7a2db05 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 11:54:37 +0200 Subject: [PATCH 078/144] move domain mailboxes to domain detail page and only enable it if catch-all is on --- .../templates/dashboard/custom_domain.html | 49 +------------------ .../dashboard/domain_detail/info.html | 32 +++++++++++- app/dashboard/views/custom_domain.py | 37 -------------- app/dashboard/views/domain_detail.py | 44 ++++++++++++++++- 4 files changed, 76 insertions(+), 86 deletions(-) diff --git a/app/dashboard/templates/dashboard/custom_domain.html b/app/dashboard/templates/dashboard/custom_domain.html index 2e4927ca..0510aa46 100644 --- a/app/dashboard/templates/dashboard/custom_domain.html +++ b/app/dashboard/templates/dashboard/custom_domain.html @@ -62,37 +62,7 @@
Created {{ custom_domain.created_at | dt }}
{{ custom_domain.nb_alias() }} aliases. -

- - Mailboxes: - -
- - {% set domain_mailboxes=custom_domain.mailboxes %} -
- - - -
-
- -
-
- -
-
-
-
@@ -118,25 +88,10 @@ {{ new_custom_domain_form.domain(class="form-control", placeholder="my-domain.com", maxlength=128) }} {{ render_field_errors(new_custom_domain_form.domain) }}
- Please use full path domain, for ex my-subdomain.my-domain.com + Please use full path domain, for example my-domain.com + or my-subdomain.my-domain.com if you are using a subdomain.
-
- By default, aliases created with your domain are "owned" by your default - mailbox {{ current_user.default_mailbox.email }}.
- This below option allow you to choose the mailbox(es) that a new alias automatically belongs to. -
- - -
diff --git a/app/dashboard/templates/dashboard/domain_detail/info.html b/app/dashboard/templates/dashboard/domain_detail/info.html index ba70d6db..d0852635 100644 --- a/app/dashboard/templates/dashboard/domain_detail/info.html +++ b/app/dashboard/templates/dashboard/domain_detail/info.html @@ -45,11 +45,38 @@
+
+
Auto-created aliases are automatically owned by these mailboxes
+ {% set domain_mailboxes=custom_domain.mailboxes %} +
+ + + +
+
+ +
+
+ +
+
+
+
+ +

Default Display Name

Default display name for aliases created with {{ custom_domain.domain }} - unless overwritten by the alias display name. + unless overwritten by the alias display name.
@@ -108,6 +135,8 @@ {% block script %} {% endblock %} + diff --git a/app/dashboard/views/custom_domain.py b/app/dashboard/views/custom_domain.py index 7bf87509..a386b50e 100644 --- a/app/dashboard/views/custom_domain.py +++ b/app/dashboard/views/custom_domain.py @@ -88,43 +88,6 @@ def custom_domain(): custom_domain_id=new_custom_domain.id, ) ) - elif request.form.get("form-name") == "update": - domain_id = request.form.get("domain-id") - domain = CustomDomain.get(domain_id) - - if not domain or domain.user_id != current_user.id: - flash("Unknown error. Refresh the page", "warning") - return redirect(url_for("dashboard.custom_domain")) - - mailbox_ids = request.form.getlist("mailbox_ids") - # check if mailbox is not tempered with - mailboxes = [] - for mailbox_id in mailbox_ids: - mailbox = Mailbox.get(mailbox_id) - if ( - not mailbox - or mailbox.user_id != current_user.id - or not mailbox.verified - ): - flash("Something went wrong, please retry", "warning") - return redirect(url_for("dashboard.custom_domain")) - mailboxes.append(mailbox) - - if not mailboxes: - flash("You must select at least 1 mailbox", "warning") - return redirect(url_for("dashboard.custom_domain")) - - # first remove all existing domain-mailboxes links - DomainMailbox.query.filter_by(domain_id=domain.id).delete() - db.session.flush() - - for mailbox in mailboxes: - DomainMailbox.create(domain_id=domain.id, mailbox_id=mailbox.id) - - db.session.commit() - flash(f"Domain {domain.domain} has been updated", "success") - - return redirect(url_for("dashboard.custom_domain")) return render_template( "dashboard/custom_domain.html", diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index d2513ed6..dcfeb02e 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -14,7 +14,7 @@ from app.dns_utils import ( from app.email_utils import send_email from app.extensions import db from app.log import LOG -from app.models import CustomDomain, Alias, DomainDeletedAlias +from app.models import CustomDomain, Alias, DomainDeletedAlias, Mailbox, DomainMailbox from app.utils import random_string @@ -159,6 +159,8 @@ def domain_detail_dns(custom_domain_id): @login_required def domain_detail(custom_domain_id): custom_domain = CustomDomain.get(custom_domain_id) + mailboxes = current_user.mailboxes() + if not custom_domain or custom_domain.user_id != current_user.id: flash("You cannot see this page", "warning") return redirect(url_for("dashboard.index")) @@ -219,6 +221,46 @@ def domain_detail(custom_domain_id): return redirect( url_for("dashboard.domain_detail", custom_domain_id=custom_domain.id) ) + elif request.form.get("form-name") == "update": + mailbox_ids = request.form.getlist("mailbox_ids") + # check if mailbox is not tempered with + mailboxes = [] + for mailbox_id in mailbox_ids: + mailbox = Mailbox.get(mailbox_id) + if ( + not mailbox + or mailbox.user_id != current_user.id + or not mailbox.verified + ): + flash("Something went wrong, please retry", "warning") + return redirect( + url_for( + "dashboard.domain_detail", custom_domain_id=custom_domain.id + ) + ) + mailboxes.append(mailbox) + + if not mailboxes: + flash("You must select at least 1 mailbox", "warning") + return redirect( + url_for( + "dashboard.domain_detail", custom_domain_id=custom_domain.id + ) + ) + + # first remove all existing domain-mailboxes links + DomainMailbox.query.filter_by(domain_id=custom_domain.id).delete() + db.session.flush() + + for mailbox in mailboxes: + DomainMailbox.create(domain_id=custom_domain.id, mailbox_id=mailbox.id) + + db.session.commit() + flash(f"{custom_domain.domain} mailboxes has been updated", "success") + + return redirect( + url_for("dashboard.domain_detail", custom_domain_id=custom_domain.id) + ) elif request.form.get("form-name") == "delete": name = custom_domain.domain LOG.d("Schedule deleting %s", custom_domain) From 58d36e9cd8714f69fa9282cd884cbec400019ca6 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 17:35:36 +0200 Subject: [PATCH 079/144] add CustomDomain.auto_create_regex column --- app/models.py | 3 ++ .../versions/2021_091716_d8c55e79da54_.py | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 migrations/versions/2021_091716_d8c55e79da54_.py diff --git a/app/models.py b/app/models.py index de1972b4..49517a56 100644 --- a/app/models.py +++ b/app/models.py @@ -1811,6 +1811,9 @@ class CustomDomain(db.Model, ModelMixin): # an alias is created automatically the first time it receives an email catch_all = db.Column(db.Boolean, nullable=False, default=False, server_default="0") + # an alias is auto created if it matches the regex + auto_create_regex = db.Column(db.String(512), nullable=True) + # option to generate random prefix version automatically random_prefix_generation = db.Column( db.Boolean, nullable=False, default=False, server_default="0" diff --git a/migrations/versions/2021_091716_d8c55e79da54_.py b/migrations/versions/2021_091716_d8c55e79da54_.py new file mode 100644 index 00000000..c9c57c6a --- /dev/null +++ b/migrations/versions/2021_091716_d8c55e79da54_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: d8c55e79da54 +Revises: 4d3f91ddf3e9 +Create Date: 2021-09-17 16:30:23.299011 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd8c55e79da54' +down_revision = '4d3f91ddf3e9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('custom_domain', sa.Column('auto_create_regex', sa.String(length=512), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('custom_domain', 'auto_create_regex') + # ### end Alembic commands ### From 0b127216ee0c2996a4de4eb6a4b38c9b9790f6ea Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 17:41:36 +0200 Subject: [PATCH 080/144] user can set custom_domain.auto_create_regex --- .../dashboard/domain_detail/info.html | 81 ++++++++++++++++--- app/dashboard/views/domain_detail.py | 22 ++++- app/models.py | 4 + static/style.css | 4 + 4 files changed, 97 insertions(+), 14 deletions(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/info.html b/app/dashboard/templates/dashboard/domain_detail/info.html index d0852635..d97f29ac 100644 --- a/app/dashboard/templates/dashboard/domain_detail/info.html +++ b/app/dashboard/templates/dashboard/domain_detail/info.html @@ -12,23 +12,13 @@
Created {{ custom_domain.created_at | dt }}. {{ nb_alias }} aliases
-

Catch All

-
- Create aliases on the fly. +

Auto create/on the fly alias

- Simply use anything@{{ custom_domain.domain }} - next time you need an alias: it'll be automatically - created the first time it receives an email.
- The new alias will belong to - {% for mailbox in custom_domain.mailboxes %} - {{ mailbox.email }} - {% if not loop.last %},{% endif %} - {% endfor %} -
+
+ + +
+ Advanced
+ You can also set a regular expression (regex): if an alias matches the expression, it'll be automatically created. +
+ Please note that only the local part of the alias (i.e. @{{ custom_domain.domain }} is ignored) during the + regex + test. + +
+ +
+ +
+ + {% if custom_domain.auto_create_regex %} + + {% endif %} +
+ + For example, if you want aliases that starts with prefix. to be automatically created, you can set the + regex to prefix\..* +
+ + If you want aliases that ends with .suffix to be automatically created, you can use the regex + .*\.suffix +
+ + To test out regex, we recommend using regex tester tool like https://regex101.com↗ +
+ +
+ The new alias will belong to + {% for mailbox in custom_domain.mailboxes %} + {{ mailbox.email }} + {% if not loop.last %},{% endif %} + {% endfor %} +
-
+
Auto-created aliases are automatically owned by these mailboxes
{% set domain_mailboxes=custom_domain.mailboxes %}
diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index dcfeb02e..ca4b4957 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -158,7 +158,7 @@ def domain_detail_dns(custom_domain_id): @dashboard_bp.route("/domains//info", methods=["GET", "POST"]) @login_required def domain_detail(custom_domain_id): - custom_domain = CustomDomain.get(custom_domain_id) + custom_domain: CustomDomain = CustomDomain.get(custom_domain_id) mailboxes = current_user.mailboxes() if not custom_domain or custom_domain.user_id != current_user.id: @@ -261,6 +261,26 @@ def domain_detail(custom_domain_id): return redirect( url_for("dashboard.domain_detail", custom_domain_id=custom_domain.id) ) + elif request.form.get("form-name") == "set-auto_create_regex": + if request.form.get("action") == "save": + auto_create_regex = request.form.get("auto_create_regex") + if auto_create_regex: + custom_domain.auto_create_regex = auto_create_regex + db.session.commit() + flash("The auto create regex has been updated", "success") + else: + flash("The auto create regex cannot be empty", "error") + else: + custom_domain.auto_create_regex = None + db.session.commit() + flash( + f"The auto create regex has been has been removed", + "info", + ) + return redirect( + url_for("dashboard.domain_detail", custom_domain_id=custom_domain.id) + ) + elif request.form.get("form-name") == "delete": name = custom_domain.domain LOG.d("Schedule deleting %s", custom_domain) diff --git a/app/models.py b/app/models.py index 49517a56..da9b5596 100644 --- a/app/models.py +++ b/app/models.py @@ -1848,6 +1848,10 @@ class CustomDomain(db.Model, ModelMixin): user = db.relationship(User, foreign_keys=[user_id]) + @property + def auto_create_alias_enabled(self) -> bool: + return self.catch_all or self.auto_create_regex is not None + @property def mailboxes(self): if self._mailboxes: diff --git a/static/style.css b/static/style.css index 18ba81da..dd196eac 100644 --- a/static/style.css +++ b/static/style.css @@ -169,4 +169,8 @@ textarea.parsley-error { .domain_detail_content { font-size: 15px; +} + +.domain_detail_content .parsley-errors-list { + max-width: 20em; } \ No newline at end of file From 2b84168d688f3dab2df2d406b6fd58f1dd286f4f Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 17:42:16 +0200 Subject: [PATCH 081/144] validate address in try_auto_create --- app/alias_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/alias_utils.py b/app/alias_utils.py index a5c4cd81..37464953 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -1,6 +1,7 @@ import re from typing import Optional +from email_validator import validate_email, EmailNotValidError from sqlalchemy.exc import IntegrityError, DataError from app.config import BOUNCE_PREFIX_FOR_REPLY_PHASE @@ -10,6 +11,7 @@ from app.email_utils import ( send_cannot_create_domain_alias, can_create_directory_for_address, send_cannot_create_directory_alias_disabled, + get_email_local_part, ) from app.errors import AliasInTrashError from app.extensions import db @@ -34,6 +36,12 @@ def try_auto_create(address: str) -> Optional[Alias]: LOG.e("alias %s can't start with %s", address, BOUNCE_PREFIX_FOR_REPLY_PHASE) return None + try: + # NOT allow unicode for now + validate_email(address, check_deliverability=False, allow_smtputf8=False) + except EmailNotValidError: + return None + alias = try_auto_create_catch_all_domain(address) if not alias: alias = try_auto_create_directory(address) From a6c874e914572ffbe5394f43c877f6963dc3d10a Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 17:42:44 +0200 Subject: [PATCH 082/144] Use validate_email in get_email_local_part and get_email_domain_part --- app/email_utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index ac219f6b..309d38bf 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -19,7 +19,7 @@ from typing import Tuple, List, Optional, Union import arrow import dkim import spf -from email_validator import validate_email, EmailNotValidError +from email_validator import validate_email, EmailNotValidError, ValidatedEmail from flanker.addresslib import address from flanker.addresslib.address import EmailAddress from jinja2 import Environment, FileSystemLoader @@ -379,8 +379,12 @@ def get_email_local_part(address) -> str: """ Get the local part from email ab@cd.com -> ab + Convert the local part to lowercase """ - return address[: address.find("@")] + r: ValidatedEmail = validate_email( + address, check_deliverability=False, allow_smtputf8=False + ) + return r.local_part.lower() def get_email_domain_part(address): @@ -388,8 +392,10 @@ def get_email_domain_part(address): Get the domain part from email ab@cd.com -> cd.com """ - address = sanitize_email(address) - return address[address.find("@") + 1 :] + r: ValidatedEmail = validate_email( + address, check_deliverability=False, allow_smtputf8=False + ) + return r.domain # headers used to DKIM sign in order of preference From 344f8e67d28bf7f9f6c3ab9c4fec9bd5f3bd823d Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 17:43:12 +0200 Subject: [PATCH 083/144] take into account custom_domain.auto_create_regex in try_auto_create_catch_all_domain() --- app/alias_utils.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index 37464953..8b67402c 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -133,10 +133,22 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: return None # custom_domain exists - if not custom_domain.catch_all: + if not custom_domain.catch_all and not custom_domain.auto_create_regex: return None - # custom_domain has catch-all enabled + if custom_domain.auto_create_regex: + local = get_email_local_part(address) + regex = re.compile(custom_domain.auto_create_regex) + if not re.fullmatch(regex, local): + LOG.d( + "%s can't be auto created on %s as it fails regex %s", + address, + custom_domain, + custom_domain.auto_create_regex, + ) + return None + + # custom_domain has catch-all enabled or the address passes the regex domain_user: User = custom_domain.user if not domain_user.can_create_new_alias(): From de3207ac4bf24b9321b29f82eb7cd99c39603834 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 18:05:18 +0200 Subject: [PATCH 084/144] Add ALIAS_AUTOMATIC_DISABLE --- app/config.py | 3 +++ example.env | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 54b36a97..2553e3f0 100644 --- a/app/config.py +++ b/app/config.py @@ -400,3 +400,6 @@ POSTMASTER = os.environ.get("POSTMASTER") # store temporary files, especially for debugging TEMP_DIR = os.environ.get("TEMP_DIR") + +# enable the alias automation disable: an alias can be automatically disabled if it has too many bounces +ALIAS_AUTOMATIC_DISABLE = "ALIAS_AUTOMATIC_DISABLE" in os.environ diff --git a/example.env b/example.env index 12279ed1..119490b3 100644 --- a/example.env +++ b/example.env @@ -175,4 +175,6 @@ DISABLE_ONBOARDING=true # POSTMASTER = postmaster@example.com -# TEMP_DIR = /tmp \ No newline at end of file +# TEMP_DIR = /tmp + +#ALIAS_AUTOMATIC_DISABLE=true \ No newline at end of file From 8b676bc4af911463024b1b58cdd19caba85c0d65 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Fri, 17 Sep 2021 18:05:35 +0200 Subject: [PATCH 085/144] not disable alias if ALIAS_AUTOMATIC_DISABLE is not set --- app/email_utils.py | 4 ++++ tests/test.env | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/email_utils.py b/app/email_utils.py index 309d38bf..7e4e1f05 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -47,6 +47,7 @@ from app.config import ( ALERT_SPF, POSTFIX_PORT_FORWARD, TEMP_DIR, + ALIAS_AUTOMATIC_DISABLE, ) from app.dns_utils import get_mx_domains from app.extensions import db @@ -1022,6 +1023,9 @@ def should_disable(alias: Alias) -> bool: LOG.w("%s cannot be disabled", alias) return False + if not ALIAS_AUTOMATIC_DISABLE: + return False + yesterday = arrow.now().shift(days=-1) nb_bounced_last_24h = ( db.session.query(EmailLog) diff --git a/tests/test.env b/tests/test.env index d1fbe886..f0a261c9 100644 --- a/tests/test.env +++ b/tests/test.env @@ -49,4 +49,6 @@ GOOGLE_CLIENT_SECRET=to_fill FACEBOOK_CLIENT_ID=to_fill FACEBOOK_CLIENT_SECRET=to_fill -PGP_SENDER_PRIVATE_KEY_PATH=local_data/private-pgp.asc \ No newline at end of file +PGP_SENDER_PRIVATE_KEY_PATH=local_data/private-pgp.asc + +ALIAS_AUTOMATIC_DISABLE=true \ No newline at end of file From ec2812bfa47c25c3307154b764224e2b9e6a05cc Mon Sep 17 00:00:00 2001 From: Son Date: Sat, 18 Sep 2021 18:46:26 +0200 Subject: [PATCH 086/144] handle invalid email in email_can_be_used_as_mailbox --- app/email_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 7e4e1f05..977ffa99 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -19,7 +19,7 @@ from typing import Tuple, List, Optional, Union import arrow import dkim import spf -from email_validator import validate_email, EmailNotValidError, ValidatedEmail +from email_validator import validate_email, EmailNotValidError, ValidatedEmail, EmailSyntaxError from flanker.addresslib import address from flanker.addresslib.address import EmailAddress from jinja2 import Environment, FileSystemLoader @@ -520,7 +520,11 @@ def email_can_be_used_as_mailbox(email: str) -> bool: - one of custom domains - a disposable domain """ - domain = get_email_domain_part(email) + try: + domain = get_email_domain_part(email) + except EmailNotValidError: + return False + if not domain: return False From 9b89d7cc5d386899e72aa7f10c0f503d014a7dda Mon Sep 17 00:00:00 2001 From: Son Date: Sat, 18 Sep 2021 18:48:31 +0200 Subject: [PATCH 087/144] update package --- app/email_utils.py | 9 +++++++-- poetry.lock | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 977ffa99..ecb6c57e 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -19,7 +19,12 @@ from typing import Tuple, List, Optional, Union import arrow import dkim import spf -from email_validator import validate_email, EmailNotValidError, ValidatedEmail, EmailSyntaxError +from email_validator import ( + validate_email, + EmailNotValidError, + ValidatedEmail, + EmailSyntaxError, +) from flanker.addresslib import address from flanker.addresslib.address import EmailAddress from jinja2 import Environment, FileSystemLoader @@ -524,7 +529,7 @@ def email_can_be_used_as_mailbox(email: str) -> bool: domain = get_email_domain_part(email) except EmailNotValidError: return False - + if not domain: return False diff --git a/poetry.lock b/poetry.lock index 8f9f44d3..a4a83025 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2416,8 +2416,11 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.8.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94"}, {file = "psycopg2_binary-2.8.6-cp38-cp38-win32.whl", hash = "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729"}, {file = "psycopg2_binary-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-macosx_10_9_x86_64.macosx_10_9_intel.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83"}, {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52"}, {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-win32.whl", hash = "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6"}, ] ptyprocess = [ {file = "ptyprocess-0.6.0-py2.py3-none-any.whl", hash = "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"}, @@ -2572,6 +2575,8 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ @@ -2595,6 +2600,12 @@ regex = [ {file = "regex-2020.9.27-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:8d69cef61fa50c8133382e61fd97439de1ae623fe943578e477e76a9d9471637"}, {file = "regex-2020.9.27-cp38-cp38-win32.whl", hash = "sha256:f2388013e68e750eaa16ccbea62d4130180c26abb1d8e5d584b9baf69672b30f"}, {file = "regex-2020.9.27-cp38-cp38-win_amd64.whl", hash = "sha256:4318d56bccfe7d43e5addb272406ade7a2274da4b70eb15922a071c58ab0108c"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux1_i686.whl", hash = "sha256:84cada8effefe9a9f53f9b0d2ba9b7b6f5edf8d2155f9fdbe34616e06ececf81"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:816064fc915796ea1f26966163f6845de5af78923dfcecf6551e095f00983650"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:5d892a4f1c999834eaa3c32bc9e8b976c5825116cde553928c4c8e7e48ebda67"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c9443124c67b1515e4fe0bb0aa18df640965e1030f468a2a5dc2589b26d130ad"}, + {file = "regex-2020.9.27-cp39-cp39-win32.whl", hash = "sha256:49f23ebd5ac073765ecbcf046edc10d63dcab2f4ae2bce160982cb30df0c0302"}, + {file = "regex-2020.9.27-cp39-cp39-win_amd64.whl", hash = "sha256:3d20024a70b97b4f9546696cbf2fd30bae5f42229fbddf8661261b1eaff0deb7"}, {file = "regex-2020.9.27.tar.gz", hash = "sha256:a6f32aea4260dfe0e55dc9733ea162ea38f0ea86aa7d0f77b15beac5bf7b369d"}, ] requests = [ @@ -2622,20 +2633,29 @@ rsa = [ {file = "ruamel.yaml.clib-0.2.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b"}, + {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win32.whl", hash = "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988"}, + {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win32.whl", hash = "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e"}, + {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win32.whl", hash = "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99"}, + {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win32.whl", hash = "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b"}, + {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a"}, + {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5"}, + {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c"}, + {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win32.whl", hash = "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd"}, + {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb"}, {file = "ruamel.yaml.clib-0.2.2.tar.gz", hash = "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7"}, ] s3transfer = [ @@ -2762,19 +2782,28 @@ typed-ast = [ {file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"}, + {file = "typed_ast-1.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f"}, {file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"}, {file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"}, {file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"}, + {file = "typed_ast-1.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298"}, {file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"}, {file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"}, {file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"}, + {file = "typed_ast-1.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d"}, {file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"}, {file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"}, {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, + {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d"}, + {file = "typed_ast-1.4.1-cp39-cp39-win32.whl", hash = "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395"}, + {file = "typed_ast-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c"}, {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] typing-extensions = [ From 838f1dc86d7643f74bbe53811ef8360c7915da74 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sat, 18 Sep 2021 19:16:43 +0200 Subject: [PATCH 088/144] add beta mention to auto create alias regex --- app/dashboard/templates/dashboard/domain_detail/info.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/info.html b/app/dashboard/templates/dashboard/domain_detail/info.html index d97f29ac..6ea287fa 100644 --- a/app/dashboard/templates/dashboard/domain_detail/info.html +++ b/app/dashboard/templates/dashboard/domain_detail/info.html @@ -49,7 +49,8 @@ {% if custom_domain.catch_all %} disabled-content {% endif %} border border-info p-2" id="regex-section"> - Advanced
+ Advanced + Beta
You can also set a regular expression (regex): if an alias matches the expression, it'll be automatically created.
Please note that only the local part of the alias (i.e. @{{ custom_domain.domain }} is ignored) during the From cbc20dd268c71dcfba089500673ea10a4acd4130 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sat, 18 Sep 2021 19:19:17 +0200 Subject: [PATCH 089/144] remove unused import --- app/email_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/email_utils.py b/app/email_utils.py index ecb6c57e..b03ba831 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -23,7 +23,6 @@ from email_validator import ( validate_email, EmailNotValidError, ValidatedEmail, - EmailSyntaxError, ) from flanker.addresslib import address from flanker.addresslib.address import EmailAddress From 530bc8591eb75bb8ef2fdc3e1b07d70fe2aa8650 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sun, 19 Sep 2021 09:36:19 +0200 Subject: [PATCH 090/144] support | in auto create alias regex --- app/dashboard/templates/dashboard/domain_detail/info.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/info.html b/app/dashboard/templates/dashboard/domain_detail/info.html index 6ea287fa..8eb97aa9 100644 --- a/app/dashboard/templates/dashboard/domain_detail/info.html +++ b/app/dashboard/templates/dashboard/domain_detail/info.html @@ -64,7 +64,7 @@ value="{{ custom_domain.auto_create_regex or "" }}" name="auto_create_regex" required - data-parsley-pattern="[0-9a-z-_.(\\)(\*)]{1,}" + data-parsley-pattern="[0-9a-z-_.(\\)(\*)(\|)]{1,}" data-parsley-trigger="change" data-parsley-error-message="Only lowercase letter, number, dot (.), dash (-), underscore (_), backslash (\), star (*) are currently supported." placeholder="prefix\..*"> From 98b472d92539c8376e813b385ec7232a524c10c5 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Sun, 19 Sep 2021 19:50:50 +0200 Subject: [PATCH 091/144] use "pin" instead of "favorite" --- app/dashboard/templates/dashboard/index.html | 6 +++--- static/js/index.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index 2035f550..7f85958f 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -264,7 +264,7 @@ {% if alias.pinned %} + title="This alias is pinned"> {% endif %} {% if alias.hibp_breaches | length > 0 %} @@ -475,8 +475,8 @@ {% endif %}
- Add to favorite + title="it's always pinned on top"> + Pin this alias
diff --git a/static/js/index.js b/static/js/index.js index dab6b406..9a34921d 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -139,9 +139,9 @@ $(".pin-alias").change(async function () { if (res.ok) { if (newValue) { - toastr.success(`${alias} is added to favorite`); + toastr.success(`${alias} is pinned`); } else { - toastr.info(`${alias} is removed from favorite`); + toastr.info(`${alias} is unpinned`); } } else { toastr.error("Sorry for the inconvenience! Could you refresh the page & retry please?", "Unknown Error"); From 7bb54e1e8e7aadb2a1a3cab6a1ac93058e2d5f3c Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 09:48:06 +0200 Subject: [PATCH 092/144] add "Custom/number_incoming_email" in newrelic --- email_handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/email_handler.py b/email_handler.py index ae531297..e1484620 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1945,6 +1945,9 @@ class MailHandler: newrelic.agent.record_custom_metric( "Custom/email_handler_time", elapsed, newrelic_app ) + newrelic.agent.record_custom_metric( + "Custom/number_incoming_email", 1, newrelic_app + ) return ret From 153831ed1a841cbe0468a314913c72d41175661c Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 12:27:36 +0200 Subject: [PATCH 093/144] remove obsolete sender formats --- app/models.py | 33 ++++++++++++--------------------- tests/api/test_setting.py | 4 ++-- tests/test_models.py | 9 ++------- 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/app/models.py b/app/models.py index da9b5596..099fb09f 100644 --- a/app/models.py +++ b/app/models.py @@ -160,9 +160,7 @@ class PlanEnum(EnumE): # Specify the format for sender address class SenderFormatEnum(EnumE): AT = 0 # John Wick - john at wick.com - VIA = 1 # john@wick.com via SimpleLogin A = 2 # John Wick - john(a)wick.com - FULL = 3 # John Wick - john@wick.com class AliasGeneratorEnum(EnumE): @@ -1486,26 +1484,19 @@ class Contact(db.Model, ModelMixin): `new_email` is a special reply address """ user = self.user - if ( - not user - or not SenderFormatEnum.has_value(user.sender_format) - or user.sender_format == SenderFormatEnum.VIA.value - ): - new_name = f"{self.website_email} via SimpleLogin" - else: - if user.sender_format == SenderFormatEnum.AT.value: - formatted_email = self.website_email.replace("@", " at ").strip() - elif user.sender_format == SenderFormatEnum.A.value: - formatted_email = self.website_email.replace("@", "(a)").strip() - elif user.sender_format == SenderFormatEnum.FULL.value: - formatted_email = self.website_email.strip() + sender_format = user.sender_format if user else SenderFormatEnum.AT.value - # Prefix name to formatted email if available - new_name = ( - (self.name + " - " + formatted_email) - if self.name and self.name != self.website_email.strip() - else formatted_email - ) + if sender_format == SenderFormatEnum.AT.value: + formatted_email = self.website_email.replace("@", " at ").strip() + else: + formatted_email = self.website_email.replace("@", "(a)").strip() + + # Prefix name to formatted email if available + new_name = ( + (self.name + " - " + formatted_email) + if self.name and self.name != self.website_email.strip() + else formatted_email + ) new_addr = formataddr((new_name, self.reply_email)).strip() return new_addr.strip() diff --git a/tests/api/test_setting.py b/tests/api/test_setting.py index dd23f9ab..d88ebd83 100644 --- a/tests/api/test_setting.py +++ b/tests/api/test_setting.py @@ -59,9 +59,9 @@ def test_update_settings_sender_format(flask_client): r = flask_client.patch("/api/setting", json={"sender_format": "invalid"}) assert r.status_code == 400 - r = flask_client.patch("/api/setting", json={"sender_format": "VIA"}) + r = flask_client.patch("/api/setting", json={"sender_format": "A"}) assert r.status_code == 200 - assert user.sender_format == SenderFormatEnum.VIA.value + assert user.sender_format == SenderFormatEnum.A.value def test_get_setting_domains(flask_client): diff --git a/tests/test_models.py b/tests/test_models.py index 48aa0af5..6bcf4ca9 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -133,17 +133,12 @@ def test_new_addr(flask_client): name="First Last", commit=True, ) - assert c1.new_addr() == '"abcd@example.com via SimpleLogin" ' - - # set sender format = FULL - user.sender_format = SenderFormatEnum.FULL.value - db.session.commit() - assert c1.new_addr() == '"First Last - abcd@example.com" ' + assert c1.new_addr() == '"First Last - abcd(a)example.com" ' # Make sure email isn't duplicated if sender name equals email c1.name = "abcd@example.com" db.session.commit() - assert c1.new_addr() == '"abcd@example.com" ' + assert c1.new_addr() == '"abcd(a)example.com" ' # set sender_format = AT user.sender_format = SenderFormatEnum.AT.value From 0aa3dff38bd915f227cb961cf39b204109492bc1 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 12:28:12 +0200 Subject: [PATCH 094/144] handle case pg_trgm can't be dropped when running test --- tests/conftest.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 97d24f61..3966ee7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,10 +2,14 @@ import os # use the tests/test.env config fle # flake8: noqa: E402 +import sqlalchemy + os.environ["CONFIG"] = os.path.abspath( os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests/test.env") ) +from psycopg2 import errors +from psycopg2.errorcodes import DEPENDENT_OBJECTS_STILL_EXIST import pytest @@ -21,8 +25,13 @@ app.config["SERVER_NAME"] = "sl.test" with app.app_context(): # enable pg_trgm extension with db.engine.connect() as conn: - conn.execute("DROP EXTENSION if exists pg_trgm") - conn.execute("CREATE EXTENSION pg_trgm") + try: + conn.execute("DROP EXTENSION if exists pg_trgm") + conn.execute("CREATE EXTENSION pg_trgm") + except sqlalchemy.exc.InternalError as e: + if isinstance(e.orig, errors.lookup(DEPENDENT_OBJECTS_STILL_EXIST)): + print(">>> pg_trgm can't be dropped, ignore") + conn.execute("Rollback") db.create_all() From 005a760710420ee8de6af0959865f2a0ffee0ab6 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 12:32:39 +0200 Subject: [PATCH 095/144] handle case catch_all is enabled but custom_domain.auto_create_regex is already set --- app/alias_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index 8b67402c..e90bdc1c 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -132,11 +132,10 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: if not custom_domain: return None - # custom_domain exists if not custom_domain.catch_all and not custom_domain.auto_create_regex: return None - if custom_domain.auto_create_regex: + if custom_domain.auto_create_regex and not custom_domain.catch_all: local = get_email_local_part(address) regex = re.compile(custom_domain.auto_create_regex) if not re.fullmatch(regex, local): From cfb7b7cefc84c86e71d03962540af5c16905878e Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 13:43:54 +0200 Subject: [PATCH 096/144] fix setting --- app/dashboard/templates/dashboard/setting.html | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/app/dashboard/templates/dashboard/setting.html b/app/dashboard/templates/dashboard/setting.html index ed3cb21b..e3696bf0 100644 --- a/app/dashboard/templates/dashboard/setting.html +++ b/app/dashboard/templates/dashboard/setting.html @@ -283,14 +283,6 @@ From 17c13ee37f8f21cfe922895341f26d04b2d0a5f0 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 13:51:16 +0200 Subject: [PATCH 097/144] ignore smtp error in handle_bounce_forward_phase() --- app/email_utils.py | 13 +++++++++++-- email_handler.py | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index b03ba831..3c672fc8 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -13,7 +13,7 @@ from email.message import Message from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import make_msgid, formatdate -from smtplib import SMTP, SMTPServerDisconnected +from smtplib import SMTP, SMTPServerDisconnected, SMTPException from typing import Tuple, List, Optional, Union import arrow @@ -315,6 +315,7 @@ def send_email_with_rate_control( html=None, max_nb_alert=MAX_ALERT_24H, nb_day=1, + ignore_smtp_error=False, ) -> bool: """Same as send_email with rate control over alert_type. Make sure no more than `max_nb_alert` emails are sent over the period of `nb_day` days @@ -341,7 +342,15 @@ def send_email_with_rate_control( SentAlert.create(user_id=user.id, alert_type=alert_type, to_email=to_email) db.session.commit() - send_email(to_email, subject, plaintext, html) + + if ignore_smtp_error: + try: + send_email(to_email, subject, plaintext, html) + except SMTPException: + LOG.w("Cannot send email to %s, subject %s", to_email, subject) + else: + send_email(to_email, subject, plaintext, html) + return True diff --git a/email_handler.py b/email_handler.py index e1484620..38f13384 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1271,6 +1271,8 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): mailbox_email=mailbox.email, ), max_nb_alert=10, + # smtp error can happen if user mailbox is unreachable, that might explain the bounce + ignore_smtp_error=True, ) else: LOG.w( @@ -1298,6 +1300,7 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): mailbox_email=mailbox.email, ), max_nb_alert=10, + ignore_smtp_error=True, ) From f6fd97ef0516a4e33c06db15f45ff30ece3707e1 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 13:54:29 +0200 Subject: [PATCH 098/144] log total number of email log for an alias when it is rate limited --- email_handler.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/email_handler.py b/email_handler.py index 38f13384..ef0a6fbd 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1850,6 +1850,17 @@ def handle(envelope: Envelope) -> str: if rate_limited(mail_from, rcpt_tos): LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos) + + # add more logging info. TODO: remove + if len(rcpt_tos) == 1: + alias = Alias.get_by(email=rcpt_tos[0]) + if alias: + LOG.w( + "total number email log on %s is %s", + alias, + EmailLog.query.filter(EmailLog.alias_id == alias.id).count(), + ) + if should_ignore_bounce(envelope.mail_from): return status.E207 else: From ba3074b94a5c0979ae9588a1a57e17a679a10a59 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 16:59:27 +0200 Subject: [PATCH 099/144] use warning instead of error --- email_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index ef0a6fbd..2469ed1a 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1615,7 +1615,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str: """return the SMTP status""" user = User.get(user_id) if not user: - LOG.e("No such user %s %s", user_id, mail_from) + LOG.w("No such user %s %s", user_id, mail_from) return status.E510 if mail_from != user.email: From 0055ca976bb685c670f375f63400c1deab5507f9 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 18:16:52 +0200 Subject: [PATCH 100/144] add flush option to ModelMixin.create() --- app/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models.py b/app/models.py index 099fb09f..42ed70e4 100644 --- a/app/models.py +++ b/app/models.py @@ -83,12 +83,17 @@ class ModelMixin(object): def create(cls, **kw): # whether should call db.session.commit commit = kw.pop("commit", False) + flush = kw.pop("flush", False) r = cls(**kw) db.session.add(r) if commit: db.session.commit() + + if flush: + db.session.flush() + return r def save(self): From f36f8b94e28aa2445c4467604431a67ba90dc98c Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 18:23:19 +0200 Subject: [PATCH 101/144] Create AutoCreateRule, AutoCreateRuleMailbox model --- app/models.py | 47 +++++++++++++++++ .../versions/2021_092018_cf1e8c1bc737_.py | 51 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 migrations/versions/2021_092018_cf1e8c1bc737_.py diff --git a/app/models.py b/app/models.py index 42ed70e4..22ff95b7 100644 --- a/app/models.py +++ b/app/models.py @@ -1875,10 +1875,57 @@ class CustomDomain(db.Model, ModelMixin): return domain + @property + def auto_create_rules(self): + return sorted(self._auto_create_rules, key=lambda rule: rule.order) + def __repr__(self): return f"" +class AutoCreateRule(db.Model, ModelMixin): + """Alias auto creation rule for custom domain""" + + __table_args__ = ( + db.UniqueConstraint( + "custom_domain_id", "order", name="uq_auto_create_rule_order" + ), + ) + + custom_domain_id = db.Column( + db.ForeignKey(CustomDomain.id, ondelete="cascade"), nullable=False + ) + # an alias is auto created if it matches the regex + regex = db.Column(db.String(512), nullable=False) + + # the order in which rules are evaluated in case there are multiple rules + order = db.Column(db.Integer, default=0, nullable=False) + + custom_domain = db.relationship(CustomDomain, backref="_auto_create_rules") + + mailboxes = db.relationship( + "Mailbox", secondary="auto_create_rule__mailbox", lazy="joined" + ) + + +class AutoCreateRuleMailbox(db.Model, ModelMixin): + """store auto create rule - mailbox association""" + + __tablename__ = "auto_create_rule__mailbox" + __table_args__ = ( + db.UniqueConstraint( + "auto_create_rule_id", "mailbox_id", name="uq_auto_create_rule_mailbox" + ), + ) + + auto_create_rule_id = db.Column( + db.ForeignKey(AutoCreateRule.id, ondelete="cascade"), nullable=False + ) + mailbox_id = db.Column( + db.ForeignKey("mailbox.id", ondelete="cascade"), nullable=False + ) + + class DomainDeletedAlias(db.Model, ModelMixin): """Store all deleted alias for a domain""" diff --git a/migrations/versions/2021_092018_cf1e8c1bc737_.py b/migrations/versions/2021_092018_cf1e8c1bc737_.py new file mode 100644 index 00000000..07510745 --- /dev/null +++ b/migrations/versions/2021_092018_cf1e8c1bc737_.py @@ -0,0 +1,51 @@ +"""empty message + +Revision ID: cf1e8c1bc737 +Revises: d8c55e79da54 +Create Date: 2021-09-20 18:14:17.798925 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'cf1e8c1bc737' +down_revision = 'd8c55e79da54' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('auto_create_rule', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('custom_domain_id', sa.Integer(), nullable=False), + sa.Column('regex', sa.String(length=512), nullable=False), + sa.Column('order', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['custom_domain_id'], ['custom_domain.id'], ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('custom_domain_id', 'order', name='uq_auto_create_rule_order') + ) + op.create_table('auto_create_rule__mailbox', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('auto_create_rule_id', sa.Integer(), nullable=False), + sa.Column('mailbox_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['auto_create_rule_id'], ['auto_create_rule.id'], ondelete='cascade'), + sa.ForeignKeyConstraint(['mailbox_id'], ['mailbox.id'], ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('auto_create_rule_id', 'mailbox_id', name='uq_auto_create_rule_mailbox') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('auto_create_rule__mailbox') + op.drop_table('auto_create_rule') + # ### end Alembic commands ### From 56c72d5fba72f3499773333b8e4b991ff788bcb9 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 18:28:43 +0200 Subject: [PATCH 102/144] create auto create page, remove custom domain auto_create_regex part --- .../dashboard/domain_detail/auto-create.html | 140 +++++++++++++++++ .../dashboard/domain_detail/base.html | 6 + .../dashboard/domain_detail/info.html | 65 +------- app/dashboard/views/domain_detail.py | 148 +++++++++++++++--- app/models.py | 4 - static/style.css | 4 - 6 files changed, 282 insertions(+), 85 deletions(-) create mode 100644 app/dashboard/templates/dashboard/domain_detail/auto-create.html diff --git a/app/dashboard/templates/dashboard/domain_detail/auto-create.html b/app/dashboard/templates/dashboard/domain_detail/auto-create.html new file mode 100644 index 00000000..7c56fc30 --- /dev/null +++ b/app/dashboard/templates/dashboard/domain_detail/auto-create.html @@ -0,0 +1,140 @@ +{% extends 'dashboard/domain_detail/base.html' %} + +{% set domain_detail_page = "auto_create" %} + +{% block title %} + {{ custom_domain.domain }} Auto Create Rules +{% endblock %} + +{% block domain_detail_content %} +

{{ custom_domain.domain }} auto create alias rules

+
+ Advanced + Beta +
+ + + + {% if custom_domain.catch_all %} +
+ Rules are ineffective when catch-all is enabled. +
+ {% endif %} + +
+
+ For a greater control than a simple catch-all, you can define a set of rules to auto create aliases.
+ A rule is based on a regular expression (regex): if an alias matches the expression, it'll be automatically + created. +
+ +
+ Only the local part of the alias (i.e. @{{ custom_domain.domain }} is ignored) during the + regex test. +
+ +
+ When there are several rules, rules will be evaluated by their order. +
+ + {% if custom_domain.auto_create_rules | length > 0 %} +
+ {% for auto_create_rule in custom_domain.auto_create_rules %} +
+
+ Order: {{ auto_create_rule.order }}
+ + New alias will belong to + {% for mailbox in auto_create_rule.mailboxes %} + {{ mailbox.email }} + {% if not loop.last %},{% endif %} + {% endfor %} + + + + + + +
+
+ {% endfor %} +
+ {% endif %} + +
+
+

New rule

+ +
+ + {{ new_auto_create_rule_form.csrf_token }} + + +
+ + + {{ new_auto_create_rule_form.regex(class="form-control", + placeholder="prefix\..*", + data_parsley_pattern="[0-9a-z-_.(\\\)(\*)(\|)]{1,}", + data_parsley_trigger="change", + data_parsley_error_message="Only lowercase letter, number, dot (.), dash (-), underscore (_), backslash (\), star (*) are currently supported.") }} + {{ render_field_errors(new_auto_create_rule_form.regex) }} + +
+ For example, if you want aliases that starts with prefix. to be automatically created, you can set + the + regex to prefix\..* +
+ + If you want aliases that ends with .suffix to be automatically created, you can use the regex + .*\.suffix +
+ To test out regex, we recommend using regex tester tool like + https://regex101.com↗ +
+
+ +
+ + {{ new_auto_create_rule_form.order(class="form-control", placeholder="10", min=1, value=1, type="number") }} + {{ render_field_errors(new_auto_create_rule_form.order) }} +
+ +
+
+ +
+
+ + +
+ +
+ +
+ + +{% endblock %} + +{% block script %} + +{% endblock %} + diff --git a/app/dashboard/templates/dashboard/domain_detail/base.html b/app/dashboard/templates/dashboard/domain_detail/base.html index 4e9112bd..21295a0a 100644 --- a/app/dashboard/templates/dashboard/domain_detail/base.html +++ b/app/dashboard/templates/dashboard/domain_detail/base.html @@ -20,6 +20,12 @@ class="list-group-item list-group-item-action {{ 'active' if domain_detail_page == 'trash' }}"> Deleted Alias + + + Auto Create + +
diff --git a/app/dashboard/templates/dashboard/domain_detail/info.html b/app/dashboard/templates/dashboard/domain_detail/info.html index 8eb97aa9..5d860010 100644 --- a/app/dashboard/templates/dashboard/domain_detail/info.html +++ b/app/dashboard/templates/dashboard/domain_detail/info.html @@ -41,68 +41,19 @@ Simply use anything@{{ custom_domain.domain }} next time you need an alias: it'll be automatically created the first time it receives an email. - To have more fine-grained control, you can also use the - regular expression . + To have more fine-grained control, you can also define + auto create + rules + .
-
- Advanced - Beta
- You can also set a regular expression (regex): if an alias matches the expression, it'll be automatically created. -
- Please note that only the local part of the alias (i.e. @{{ custom_domain.domain }} is ignored) during the - regex - test. -
- -
- -
- - {% if custom_domain.auto_create_regex %} - - {% endif %} -
- - For example, if you want aliases that starts with prefix. to be automatically created, you can set the - regex to prefix\..* -
- - If you want aliases that ends with .suffix to be automatically created, you can use the regex - .*\.suffix -
- - To test out regex, we recommend using regex tester tool like https://regex101.com↗ -
- -
- The new alias will belong to - {% for mailbox in custom_domain.mailboxes %} - {{ mailbox.email }} - {% if not loop.last %},{% endif %} - {% endfor %} -
-
-
Auto-created aliases are automatically owned by these mailboxes
+
+
Auto-created aliases are automatically owned by the following mailboxes + . +
{% set domain_mailboxes=custom_domain.mailboxes %}
diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index ca4b4957..10096e81 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -2,6 +2,8 @@ from threading import Thread from flask import render_template, request, redirect, url_for, flash from flask_login import login_required, current_user +from flask_wtf import FlaskForm +from wtforms import StringField, validators, IntegerField from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN from app.dashboard.base import dashboard_bp @@ -14,7 +16,15 @@ from app.dns_utils import ( from app.email_utils import send_email from app.extensions import db from app.log import LOG -from app.models import CustomDomain, Alias, DomainDeletedAlias, Mailbox, DomainMailbox +from app.models import ( + CustomDomain, + Alias, + DomainDeletedAlias, + Mailbox, + DomainMailbox, + AutoCreateRule, + AutoCreateRuleMailbox, +) from app.utils import random_string @@ -261,25 +271,6 @@ def domain_detail(custom_domain_id): return redirect( url_for("dashboard.domain_detail", custom_domain_id=custom_domain.id) ) - elif request.form.get("form-name") == "set-auto_create_regex": - if request.form.get("action") == "save": - auto_create_regex = request.form.get("auto_create_regex") - if auto_create_regex: - custom_domain.auto_create_regex = auto_create_regex - db.session.commit() - flash("The auto create regex has been updated", "success") - else: - flash("The auto create regex cannot be empty", "error") - else: - custom_domain.auto_create_regex = None - db.session.commit() - flash( - f"The auto create regex has been has been removed", - "info", - ) - return redirect( - url_for("dashboard.domain_detail", custom_domain_id=custom_domain.id) - ) elif request.form.get("form-name") == "delete": name = custom_domain.domain @@ -378,3 +369,120 @@ def domain_detail_trash(custom_domain_id): domain_deleted_aliases=domain_deleted_aliases, custom_domain=custom_domain, ) + + +class AutoCreateRuleForm(FlaskForm): + regex = StringField( + "regex", validators=[validators.DataRequired(), validators.Length(max=128)] + ) + + order = IntegerField( + "order", + validators=[validators.DataRequired(), validators.NumberRange(min=0, max=100)], + ) + + +@dashboard_bp.route( + "/domains//auto-create", methods=["GET", "POST"] +) +@login_required +def domain_detail_auto_create(custom_domain_id): + custom_domain: CustomDomain = CustomDomain.get(custom_domain_id) + mailboxes = current_user.mailboxes() + new_auto_create_rule_form = AutoCreateRuleForm() + + if not custom_domain or custom_domain.user_id != current_user.id: + flash("You cannot see this page", "warning") + return redirect(url_for("dashboard.index")) + + if request.method == "POST": + if request.form.get("form-name") == "create-auto-create-rule": + if new_auto_create_rule_form.validate(): + # make sure order isn't used before + for auto_create_rule in custom_domain.auto_create_rules: + auto_create_rule: AutoCreateRule + if auto_create_rule.order == int( + new_auto_create_rule_form.order.data + ): + flash( + "Another rule with the same order already exists", "error" + ) + break + else: + mailbox_ids = request.form.getlist("mailbox_ids") + # check if mailbox is not tempered with + mailboxes = [] + for mailbox_id in mailbox_ids: + mailbox = Mailbox.get(mailbox_id) + if ( + not mailbox + or mailbox.user_id != current_user.id + or not mailbox.verified + ): + flash("Something went wrong, please retry", "warning") + return redirect( + url_for( + "dashboard.domain_detail_auto_create", + custom_domain_id=custom_domain.id, + ) + ) + mailboxes.append(mailbox) + + if not mailboxes: + flash("You must select at least 1 mailbox", "warning") + return redirect( + url_for( + "dashboard.domain_detail_auto_create", + custom_domain_id=custom_domain.id, + ) + ) + + rule = AutoCreateRule.create( + custom_domain_id=custom_domain.id, + order=int(new_auto_create_rule_form.order.data), + regex=new_auto_create_rule_form.regex.data, + flush=True, + ) + + for mailbox in mailboxes: + AutoCreateRuleMailbox.create( + auto_create_rule_id=rule.id, mailbox_id=mailbox.id + ) + + db.session.commit() + + flash("New auto create rule has been created", "success") + + return redirect( + url_for( + "dashboard.domain_detail_auto_create", + custom_domain_id=custom_domain.id, + ) + ) + elif request.form.get("form-name") == "delete-auto-create-rule": + rule_id = request.form.get("rule-id") + rule: AutoCreateRule = AutoCreateRule.get(int(rule_id)) + + if not rule or rule.custom_domain_id != custom_domain.id: + flash("Something wrong, please retry", "error") + return redirect( + url_for( + "dashboard.domain_detail_auto_create", + custom_domain_id=custom_domain.id, + ) + ) + + rule_order = rule.order + AutoCreateRule.delete(rule_id) + db.session.commit() + flash(f"Rule #{rule_order} has been deleted", "success") + + return redirect( + url_for( + "dashboard.domain_detail_auto_create", custom_domain_id=custom_domain.id + ) + ) + + nb_alias = Alias.filter_by(custom_domain_id=custom_domain.id).count() + + return render_template("dashboard/domain_detail/auto-create.html", **locals()) diff --git a/app/models.py b/app/models.py index 22ff95b7..7fc862dc 100644 --- a/app/models.py +++ b/app/models.py @@ -1844,10 +1844,6 @@ class CustomDomain(db.Model, ModelMixin): user = db.relationship(User, foreign_keys=[user_id]) - @property - def auto_create_alias_enabled(self) -> bool: - return self.catch_all or self.auto_create_regex is not None - @property def mailboxes(self): if self._mailboxes: diff --git a/static/style.css b/static/style.css index dd196eac..0996d792 100644 --- a/static/style.css +++ b/static/style.css @@ -170,7 +170,3 @@ textarea.parsley-error { .domain_detail_content { font-size: 15px; } - -.domain_detail_content .parsley-errors-list { - max-width: 20em; -} \ No newline at end of file From e5a8ce1492a84b59a7f4b20e6c7aed99db4cc2c5 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 20 Sep 2021 18:29:36 +0200 Subject: [PATCH 103/144] use AutoCreateRule instead of custom_domain.auto_create_regex when creating new alias --- app/alias_utils.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index e90bdc1c..238cef67 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -27,6 +27,7 @@ from app.models import ( Mailbox, EmailLog, Contact, + AutoCreateRule, ) @@ -132,22 +133,28 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: if not custom_domain: return None - if not custom_domain.catch_all and not custom_domain.auto_create_regex: + if not custom_domain.catch_all and len(custom_domain.auto_create_rules) == 0: return None - - if custom_domain.auto_create_regex and not custom_domain.catch_all: + elif not custom_domain.catch_all and len(custom_domain.auto_create_rules) > 0: local = get_email_local_part(address) - regex = re.compile(custom_domain.auto_create_regex) - if not re.fullmatch(regex, local): - LOG.d( - "%s can't be auto created on %s as it fails regex %s", - address, - custom_domain, - custom_domain.auto_create_regex, - ) - return None + for rule in custom_domain.auto_create_rules: + rule: AutoCreateRule + regex = re.compile(rule.regex) + if re.fullmatch(regex, local): + LOG.d( + "%s passes %s on %s", + address, + rule.regex, + custom_domain, + ) + mailboxes = rule.mailboxes + break + else: # no rule passes + LOG.d("no rule passed to create %s", local) + return + else: # catch-all is enabled + mailboxes = custom_domain.mailboxes - # custom_domain has catch-all enabled or the address passes the regex domain_user: User = custom_domain.user if not domain_user.can_create_new_alias(): @@ -156,7 +163,6 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: try: LOG.d("create alias %s for domain %s", address, custom_domain) - mailboxes = custom_domain.mailboxes alias = Alias.create( email=address, user_id=custom_domain.user_id, From 809f54774212c064e80c1b188157ea21a1535054 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 09:48:07 +0200 Subject: [PATCH 104/144] remove CustomDomain.auto_create_regex --- app/models.py | 3 -- .../versions/2021_092018_7a105bfc0cd0_.py | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 migrations/versions/2021_092018_7a105bfc0cd0_.py diff --git a/app/models.py b/app/models.py index 7fc862dc..531ffeab 100644 --- a/app/models.py +++ b/app/models.py @@ -1807,9 +1807,6 @@ class CustomDomain(db.Model, ModelMixin): # an alias is created automatically the first time it receives an email catch_all = db.Column(db.Boolean, nullable=False, default=False, server_default="0") - # an alias is auto created if it matches the regex - auto_create_regex = db.Column(db.String(512), nullable=True) - # option to generate random prefix version automatically random_prefix_generation = db.Column( db.Boolean, nullable=False, default=False, server_default="0" diff --git a/migrations/versions/2021_092018_7a105bfc0cd0_.py b/migrations/versions/2021_092018_7a105bfc0cd0_.py new file mode 100644 index 00000000..250b0f88 --- /dev/null +++ b/migrations/versions/2021_092018_7a105bfc0cd0_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: 7a105bfc0cd0 +Revises: cf1e8c1bc737 +Create Date: 2021-09-20 18:41:43.017908 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7a105bfc0cd0' +down_revision = 'cf1e8c1bc737' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('custom_domain', 'auto_create_regex') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('custom_domain', sa.Column('auto_create_regex', sa.VARCHAR(length=512), autoincrement=False, nullable=True)) + # ### end Alembic commands ### From f160ebec4e4ad9e92b199aed68d3ba4f5be7206d Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 10:14:36 +0200 Subject: [PATCH 105/144] add debug zone to auto create page --- .../dashboard/domain_detail/auto-create.html | 33 ++++++++++++++++ app/dashboard/views/domain_detail.py | 38 ++++++++++++++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/auto-create.html b/app/dashboard/templates/dashboard/domain_detail/auto-create.html index 7c56fc30..d6356d82 100644 --- a/app/dashboard/templates/dashboard/domain_detail/auto-create.html +++ b/app/dashboard/templates/dashboard/domain_detail/auto-create.html @@ -125,6 +125,39 @@
+
+
+

Debug Zone

+

You can test whether an alias will be automatically created given the rules above

+
+ No worries, no alias will be created during the test :) +
+ + + {{ auto_create_test_form.csrf_token }} + +
+
+ {{ auto_create_test_form.local(class="form-control", type="text", placeholder="local", value=auto_create_test_local) }} + {{ render_field_errors(auto_create_test_form.local) }} + @{{ custom_domain.domain }} +
+
+ +
+
+
+ +
+ + + {% if auto_create_test_result %} +
+ {{ auto_create_test_result }} +
+ {% endif %} +
+
diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index 10096e81..420b8dc9 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -1,3 +1,4 @@ +import re from threading import Thread from flask import render_template, request, redirect, url_for, flash @@ -382,6 +383,12 @@ class AutoCreateRuleForm(FlaskForm): ) +class AutoCreateTestForm(FlaskForm): + local = StringField( + "local part", validators=[validators.DataRequired(), validators.Length(max=128)] + ) + + @dashboard_bp.route( "/domains//auto-create", methods=["GET", "POST"] ) @@ -391,6 +398,13 @@ def domain_detail_auto_create(custom_domain_id): mailboxes = current_user.mailboxes() new_auto_create_rule_form = AutoCreateRuleForm() + auto_create_test_form = AutoCreateTestForm() + auto_create_test_local, auto_create_test_result, auto_create_test_passed = ( + "", + "", + False, + ) + if not custom_domain or custom_domain.user_id != current_user.id: flash("You cannot see this page", "warning") return redirect(url_for("dashboard.index")) @@ -476,6 +490,28 @@ def domain_detail_auto_create(custom_domain_id): AutoCreateRule.delete(rule_id) db.session.commit() flash(f"Rule #{rule_order} has been deleted", "success") + elif request.form.get("form-name") == "test-auto-create-rule": + if auto_create_test_form.validate(): + local = auto_create_test_form.local.data + auto_create_test_local = local + + for rule in custom_domain.auto_create_rules: + rule: AutoCreateRule + regex = re.compile(rule.regex) + if re.fullmatch(regex, local): + auto_create_test_result = ( + f"{local}@{custom_domain.domain} passes rule #{rule.order}" + ) + auto_create_test_passed = True + break + else: # no rule passes + auto_create_test_result = ( + f"{local}@{custom_domain.domain} doesn't pass any rule" + ) + + return render_template( + "dashboard/domain_detail/auto-create.html", **locals() + ) return redirect( url_for( @@ -483,6 +519,4 @@ def domain_detail_auto_create(custom_domain_id): ) ) - nb_alias = Alias.filter_by(custom_domain_id=custom_domain.id).count() - return render_template("dashboard/domain_detail/auto-create.html", **locals()) From 1e3afa257c53ef9413285414f5d47ec5c4058d83 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 10:42:02 +0200 Subject: [PATCH 106/144] validate regex before creating rule --- app/dashboard/views/domain_detail.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index 420b8dc9..8a555226 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -451,6 +451,20 @@ def domain_detail_auto_create(custom_domain_id): ) ) + try: + re.compile(new_auto_create_rule_form.regex.data) + except: + flash( + f"Invalid regex {new_auto_create_rule_form.regex.data}", + "error", + ) + return redirect( + url_for( + "dashboard.domain_detail_auto_create", + custom_domain_id=custom_domain.id, + ) + ) + rule = AutoCreateRule.create( custom_domain_id=custom_domain.id, order=int(new_auto_create_rule_form.order.data), From 84880ae32ad0defad633c26401e5018d3ae349b1 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 10:42:32 +0200 Subject: [PATCH 107/144] fix error with rule regex doesn't save if error --- app/dashboard/views/domain_detail.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index 8a555226..264cab51 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -504,6 +504,12 @@ def domain_detail_auto_create(custom_domain_id): AutoCreateRule.delete(rule_id) db.session.commit() flash(f"Rule #{rule_order} has been deleted", "success") + return redirect( + url_for( + "dashboard.domain_detail_auto_create", + custom_domain_id=custom_domain.id, + ) + ) elif request.form.get("form-name") == "test-auto-create-rule": if auto_create_test_form.validate(): local = auto_create_test_form.local.data @@ -527,10 +533,4 @@ def domain_detail_auto_create(custom_domain_id): "dashboard/domain_detail/auto-create.html", **locals() ) - return redirect( - url_for( - "dashboard.domain_detail_auto_create", custom_domain_id=custom_domain.id - ) - ) - return render_template("dashboard/domain_detail/auto-create.html", **locals()) From 2c60414796375232e7d7654060687444f5b497cb Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 10:43:02 +0200 Subject: [PATCH 108/144] when new rule is created, go to the rule list section --- .../templates/dashboard/domain_detail/auto-create.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/auto-create.html b/app/dashboard/templates/dashboard/domain_detail/auto-create.html index d6356d82..6a0280f9 100644 --- a/app/dashboard/templates/dashboard/domain_detail/auto-create.html +++ b/app/dashboard/templates/dashboard/domain_detail/auto-create.html @@ -38,7 +38,7 @@
{% if custom_domain.auto_create_rules | length > 0 %} -
+
{% for auto_create_rule in custom_domain.auto_create_rules %}
@@ -61,11 +61,11 @@
{% endif %} -
+

New rule

-
+ {{ new_auto_create_rule_form.csrf_token }} @@ -152,7 +152,7 @@
{% if auto_create_test_result %} -
+
{{ auto_create_test_result }}
{% endif %} From 1a6c68e98d7f497f75ca8ff47d4551b81a17195f Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 10:54:35 +0200 Subject: [PATCH 109/144] install pyre2 --- poetry.lock | 62 ++++++++++++++++++++++++++------------------------ pyproject.toml | 1 + 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/poetry.lock b/poetry.lock index a4a83025..90f0bea4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1257,6 +1257,18 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "pyre2" +version = "0.3.6" +description = "Python wrapper for Google\\'s RE2 using Cython" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +perf = ["regex"] +test = ["pytest"] + [[package]] name = "pyreadline" version = "2.1" @@ -1792,7 +1804,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "a2f1de1d4ecbe4b1e36d965bb9ba52d10d16b344b6c7e4509a53245b67fdcec5" +content-hash = "7c789c1e89d6b2859ee06adcc2b1caf93c759097624059461d1d613da1888e8e" [metadata.files] aiohttp = [ @@ -2416,11 +2428,8 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.8.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94"}, {file = "psycopg2_binary-2.8.6-cp38-cp38-win32.whl", hash = "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729"}, {file = "psycopg2_binary-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-macosx_10_9_x86_64.macosx_10_9_intel.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83"}, {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52"}, {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-win32.whl", hash = "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6"}, ] ptyprocess = [ {file = "ptyprocess-0.6.0-py2.py3-none-any.whl", hash = "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"}, @@ -2525,6 +2534,25 @@ pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] +pyre2 = [ + {file = "pyre2-0.3.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6d8e550899886ee01f1b8149ba1c336e1c749cec2e33414815a76fb5649cdf67"}, + {file = "pyre2-0.3.6-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:310d5c98495114692940ffa020aaeef1341427755b6ca5a17c63092060ed93dc"}, + {file = "pyre2-0.3.6-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:97de5d4cf7d8b9be7dbe0dc0941c4a6c1395fc598722d9644adc55427d3dd083"}, + {file = "pyre2-0.3.6-cp36-cp36m-win_amd64.whl", hash = "sha256:18cd5d76973ee57232a5d851489c202105e4752aee6dcbd38742c0475f3f1c4e"}, + {file = "pyre2-0.3.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:225784d7bd905bc3e87d4bbcc6ac4087ccea8905dd657273fd71bfb113e50e82"}, + {file = "pyre2-0.3.6-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:c3b45f789374d0f95866330fcd34bb6b93705e8f5c276d9d70d318a227ba5954"}, + {file = "pyre2-0.3.6-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f3467dd9a4c8100f6406bc6277d945a13b7fd7c4426d2415564de1324b5db94f"}, + {file = "pyre2-0.3.6-cp37-cp37m-win_amd64.whl", hash = "sha256:608558276d3539002ad6300d0b0a2b0941577fdea009715ff4d31052e05cb409"}, + {file = "pyre2-0.3.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ebe92a3222f2f6f176eeb3859638734e4f9a82d5940ad7d6f0c1288153c70ce2"}, + {file = "pyre2-0.3.6-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:961020835a3b805eed51a082e5effdccb51979c4efef2a17f17122967cb4749a"}, + {file = "pyre2-0.3.6-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:cc180989186f05b75020b53c79059c338e9e1940d325fc945c84aab2b5c57525"}, + {file = "pyre2-0.3.6-cp38-cp38-win_amd64.whl", hash = "sha256:7c398942c3467fe23b2dd4a11dd78da8aee774d0b481e84b1b208819ee724cca"}, + {file = "pyre2-0.3.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:617c4d75b41b34afe7590e144efad1c564a8b49a1e0827872afc2243b24beada"}, + {file = "pyre2-0.3.6-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:f3ae7b087abcbc4b910d535c2fb877ef452b61d2514a63fd15b8b020b51fe4b5"}, + {file = "pyre2-0.3.6-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:b87e9aeee74376210bd82c8328eb007b93378f3cd61fa6176161c3b9037e8474"}, + {file = "pyre2-0.3.6-cp39-cp39-win_amd64.whl", hash = "sha256:1efec117f2543b38adcbe038a2ae156eb91b6ed8a73c998c3752a766d6241075"}, + {file = "pyre2-0.3.6.tar.gz", hash = "sha256:6fe972c0cadec49a5a055690e5aa29f8aebaed0fa9b7d8d3530e33719b61f91c"}, +] pyreadline = [ {file = "pyreadline-2.1.win-amd64.exe", hash = "sha256:9ce5fa65b8992dfa373bddc5b6e0864ead8f291c94fbfec05fbd5c836162e67b"}, {file = "pyreadline-2.1.win32.exe", hash = "sha256:65540c21bfe14405a3a77e4c085ecfce88724743a4ead47c66b84defcf82c32e"}, @@ -2575,8 +2603,6 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, - {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, - {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ @@ -2600,12 +2626,6 @@ regex = [ {file = "regex-2020.9.27-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:8d69cef61fa50c8133382e61fd97439de1ae623fe943578e477e76a9d9471637"}, {file = "regex-2020.9.27-cp38-cp38-win32.whl", hash = "sha256:f2388013e68e750eaa16ccbea62d4130180c26abb1d8e5d584b9baf69672b30f"}, {file = "regex-2020.9.27-cp38-cp38-win_amd64.whl", hash = "sha256:4318d56bccfe7d43e5addb272406ade7a2274da4b70eb15922a071c58ab0108c"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux1_i686.whl", hash = "sha256:84cada8effefe9a9f53f9b0d2ba9b7b6f5edf8d2155f9fdbe34616e06ececf81"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:816064fc915796ea1f26966163f6845de5af78923dfcecf6551e095f00983650"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:5d892a4f1c999834eaa3c32bc9e8b976c5825116cde553928c4c8e7e48ebda67"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c9443124c67b1515e4fe0bb0aa18df640965e1030f468a2a5dc2589b26d130ad"}, - {file = "regex-2020.9.27-cp39-cp39-win32.whl", hash = "sha256:49f23ebd5ac073765ecbcf046edc10d63dcab2f4ae2bce160982cb30df0c0302"}, - {file = "regex-2020.9.27-cp39-cp39-win_amd64.whl", hash = "sha256:3d20024a70b97b4f9546696cbf2fd30bae5f42229fbddf8661261b1eaff0deb7"}, {file = "regex-2020.9.27.tar.gz", hash = "sha256:a6f32aea4260dfe0e55dc9733ea162ea38f0ea86aa7d0f77b15beac5bf7b369d"}, ] requests = [ @@ -2633,29 +2653,20 @@ rsa = [ {file = "ruamel.yaml.clib-0.2.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win32.whl", hash = "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win32.whl", hash = "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win32.whl", hash = "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win32.whl", hash = "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win32.whl", hash = "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb"}, {file = "ruamel.yaml.clib-0.2.2.tar.gz", hash = "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7"}, ] s3transfer = [ @@ -2782,28 +2793,19 @@ typed-ast = [ {file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"}, - {file = "typed_ast-1.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f"}, {file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"}, {file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"}, {file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"}, - {file = "typed_ast-1.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298"}, {file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"}, {file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"}, {file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"}, - {file = "typed_ast-1.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d"}, {file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"}, {file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"}, {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, - {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c"}, - {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072"}, - {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91"}, - {file = "typed_ast-1.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d"}, - {file = "typed_ast-1.4.1-cp39-cp39-win32.whl", hash = "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395"}, - {file = "typed_ast-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c"}, {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] typing-extensions = [ diff --git a/pyproject.toml b/pyproject.toml index 9c3e46c3..d4992cd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ coinbase-commerce = "^1.0.1" requests = "^2.25.1" newrelic = "^6.4.4" flanker = "^0.9.11" +pyre2 = "^0.3.6" [tool.poetry.dev-dependencies] pytest = "^6.1.0" From 1d09d76cb2dd9dc09a943220e4ad96ad8d80904a Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 10:57:36 +0200 Subject: [PATCH 110/144] use re2 instead of re to avoid ReDOS attack --- app/alias_utils.py | 2 +- app/dashboard/views/domain_detail.py | 2 +- app/dashboard/views/referral.py | 2 +- app/email_utils.py | 2 +- app/spamassassin_utils.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index 238cef67..04df5ff3 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -1,4 +1,4 @@ -import re +import re2 as re from typing import Optional from email_validator import validate_email, EmailNotValidError diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index 264cab51..8773a970 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -1,4 +1,4 @@ -import re +import re2 as re from threading import Thread from flask import render_template, request, redirect, url_for, flash diff --git a/app/dashboard/views/referral.py b/app/dashboard/views/referral.py index b1f3d837..a037cb02 100644 --- a/app/dashboard/views/referral.py +++ b/app/dashboard/views/referral.py @@ -1,4 +1,4 @@ -import re +import re2 as re from flask import render_template, request, flash, redirect, url_for from flask_login import login_required, current_user diff --git a/app/email_utils.py b/app/email_utils.py index 3c672fc8..c55667d9 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -4,7 +4,7 @@ import enum import os import quopri import random -import re +import re2 as re import time import uuid from copy import deepcopy diff --git a/app/spamassassin_utils.py b/app/spamassassin_utils.py index 05fb2739..f62a4979 100644 --- a/app/spamassassin_utils.py +++ b/app/spamassassin_utils.py @@ -2,7 +2,7 @@ https://github.com/petermat/spamassassin_client """ import logging -import re +import re2 as re import select import socket from io import BytesIO From bcc5126500954ba6cea9e4e3eb8c8aedc0417049 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 10:57:47 +0200 Subject: [PATCH 111/144] remove any restriction on regex --- .../templates/dashboard/domain_detail/auto-create.html | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/dashboard/templates/dashboard/domain_detail/auto-create.html b/app/dashboard/templates/dashboard/domain_detail/auto-create.html index 6a0280f9..e7120b43 100644 --- a/app/dashboard/templates/dashboard/domain_detail/auto-create.html +++ b/app/dashboard/templates/dashboard/domain_detail/auto-create.html @@ -74,10 +74,8 @@ {{ new_auto_create_rule_form.regex(class="form-control", - placeholder="prefix\..*", - data_parsley_pattern="[0-9a-z-_.(\\\)(\*)(\|)]{1,}", - data_parsley_trigger="change", - data_parsley_error_message="Only lowercase letter, number, dot (.), dash (-), underscore (_), backslash (\), star (*) are currently supported.") }} + placeholder="prefix\..*" + ) }} {{ render_field_errors(new_auto_create_rule_form.regex) }}
From 8d40392b5cdb3aa2b728ae8dceb2407c91bf61d1 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 11:08:27 +0200 Subject: [PATCH 112/144] allow bare "accept" in flake8 --- .flake8 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 4b631a19..57cd495c 100644 --- a/.flake8 +++ b/.flake8 @@ -6,7 +6,9 @@ ignore = W503, E203, # Ignore "f-string is missing placeholders" - F541 + F541, + # allow bare "except" + E722 exclude = .git, __pycache__, From d9d67df126af2e702f85e8f4171a842bbb06ff71 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 11:15:36 +0200 Subject: [PATCH 113/144] remove unique constraint on TransactionalEmail.email --- app/email_utils.py | 4 +-- app/models.py | 2 +- .../versions/2021_092111_bc75acacc98e_.py | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 migrations/versions/2021_092111_bc75acacc98e_.py diff --git a/app/email_utils.py b/app/email_utils.py index c55667d9..fcfa558e 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -298,9 +298,7 @@ def send_email( msg_raw = to_bytes(msg) - transaction = TransactionalEmail.get_by(email=to_email) - if not transaction: - transaction = TransactionalEmail.create(email=to_email, commit=True) + transaction = TransactionalEmail.create(email=to_email, commit=True) # use a different envelope sender for each transactional email (aka VERP) smtp.sendmail(TRANSACTIONAL_BOUNCE_EMAIL.format(transaction.id), to_email, msg_raw) diff --git a/app/models.py b/app/models.py index 531ffeab..c6107a01 100644 --- a/app/models.py +++ b/app/models.py @@ -2421,7 +2421,7 @@ class TransactionalEmail(db.Model, ModelMixin): Deleted after 7 days """ - email = db.Column(db.String(256), nullable=False, unique=True) + email = db.Column(db.String(256), nullable=False, unique=False) class Payout(db.Model, ModelMixin): diff --git a/migrations/versions/2021_092111_bc75acacc98e_.py b/migrations/versions/2021_092111_bc75acacc98e_.py new file mode 100644 index 00000000..4b662994 --- /dev/null +++ b/migrations/versions/2021_092111_bc75acacc98e_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: bc75acacc98e +Revises: 7a105bfc0cd0 +Create Date: 2021-09-21 11:15:15.573629 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'bc75acacc98e' +down_revision = '7a105bfc0cd0' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('transactional_email_email_key', 'transactional_email', type_='unique') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint('transactional_email_email_key', 'transactional_email', ['email']) + # ### end Alembic commands ### From 19088ba85f81c1b7ae6910f488851eb6ad916bdb Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 11:17:32 +0200 Subject: [PATCH 114/144] refactor --- app/dashboard/views/mailbox_detail.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/dashboard/views/mailbox_detail.py b/app/dashboard/views/mailbox_detail.py index f11326ce..cdd2bfd1 100644 --- a/app/dashboard/views/mailbox_detail.py +++ b/app/dashboard/views/mailbox_detail.py @@ -251,15 +251,15 @@ def cancel_mailbox_change_route(mailbox_id): @dashboard_bp.route("/mailbox/confirm_change") def mailbox_confirm_change_route(): s = Signer(MAILBOX_SECRET) - mailbox_id = request.args.get("mailbox_id") + signed_mailbox_id = request.args.get("mailbox_id") try: - r_id = int(s.unsign(mailbox_id)) + mailbox_id = int(s.unsign(signed_mailbox_id)) except Exception: flash("Invalid link", "error") return redirect(url_for("dashboard.index")) else: - mailbox = Mailbox.get(r_id) + mailbox = Mailbox.get(mailbox_id) # new_email can be None if user cancels change in the meantime if mailbox and mailbox.new_email: From 18acfd9a42714b5c17f86893bc902c4ed05bdfd5 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 11:19:22 +0200 Subject: [PATCH 115/144] handle case mailbox new address is already used --- app/dashboard/views/mailbox_detail.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/dashboard/views/mailbox_detail.py b/app/dashboard/views/mailbox_detail.py index cdd2bfd1..03a8a265 100644 --- a/app/dashboard/views/mailbox_detail.py +++ b/app/dashboard/views/mailbox_detail.py @@ -263,6 +263,12 @@ def mailbox_confirm_change_route(): # new_email can be None if user cancels change in the meantime if mailbox and mailbox.new_email: + if Mailbox.get_by(email=mailbox.new_email): + flash(f"{mailbox.new_email} is already used", "error") + return redirect( + url_for("dashboard.mailbox_detail_route", mailbox_id=mailbox.id) + ) + mailbox.email = mailbox.new_email mailbox.new_email = None From 65ce47b6f7d8460e9f7f1038e0f4a2cb34e5a47b Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 11:25:52 +0200 Subject: [PATCH 116/144] Allow ApiKey.name to be null --- app/models.py | 2 +- .../versions/2021_092111_b8b4f9598240_.py | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 migrations/versions/2021_092111_b8b4f9598240_.py diff --git a/app/models.py b/app/models.py index c6107a01..644d6f95 100644 --- a/app/models.py +++ b/app/models.py @@ -1762,7 +1762,7 @@ class ApiKey(db.Model, ModelMixin): user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False) code = db.Column(db.String(128), unique=True, nullable=False) - name = db.Column(db.String(128), nullable=False) + name = db.Column(db.String(128), nullable=True) last_used = db.Column(ArrowType, default=None) times = db.Column(db.Integer, default=0, nullable=False) diff --git a/migrations/versions/2021_092111_b8b4f9598240_.py b/migrations/versions/2021_092111_b8b4f9598240_.py new file mode 100644 index 00000000..ad6cde35 --- /dev/null +++ b/migrations/versions/2021_092111_b8b4f9598240_.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: b8b4f9598240 +Revises: bc75acacc98e +Create Date: 2021-09-21 11:22:24.285286 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b8b4f9598240' +down_revision = 'bc75acacc98e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('api_key', 'name', + existing_type=sa.VARCHAR(length=128), + nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('api_key', 'name', + existing_type=sa.VARCHAR(length=128), + nullable=False) + # ### end Alembic commands ### From 3c81f982caecb442cfb8e6f6a3d481aab56a8751 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 11:26:05 +0200 Subject: [PATCH 117/144] display N/A if api key name is null --- app/dashboard/templates/dashboard/api_key.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dashboard/templates/dashboard/api_key.html b/app/dashboard/templates/dashboard/api_key.html index b832d2cc..c002d769 100644 --- a/app/dashboard/templates/dashboard/api_key.html +++ b/app/dashboard/templates/dashboard/api_key.html @@ -27,7 +27,7 @@
-
{{ api_key.name }}
+
{{ api_key.name or "N/A" }}
{% if api_key.last_used %} Last used: {{ api_key.last_used | dt }}
From 0075cee1ee3abb93fbbc0a9d81eed8c63ff2b0cf Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 11:27:37 +0200 Subject: [PATCH 118/144] default to UUID for api key code if the previous one is already used --- app/models.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/app/models.py b/app/models.py index 644d6f95..8983e72d 100644 --- a/app/models.py +++ b/app/models.py @@ -1769,18 +1769,12 @@ class ApiKey(db.Model, ModelMixin): user = db.relationship(User) @classmethod - def create(cls, user_id, name): - # generate unique code - found = False - while not found: - code = random_string(60) + def create(cls, user_id, name=None, **kwargs): + code = random_string(60) + if cls.get_by(code=code): + code = str(uuid.uuid4()) - if not cls.get_by(code=code): - found = True - - a = cls(user_id=user_id, code=code, name=name) - db.session.add(a) - return a + return super().create(user_id=user_id, name=name, code=code, **kwargs) class CustomDomain(db.Model, ModelMixin): From 9279b209754afffc8406b0de19f53f081c5336be Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 12:20:29 +0200 Subject: [PATCH 119/144] only put pinned alias in default sorting --- app/api/serializer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/serializer.py b/app/api/serializer.py index f31dfee4..2128a05e 100644 --- a/app/api/serializer.py +++ b/app/api/serializer.py @@ -178,8 +178,6 @@ def get_alias_infos_with_pagination_v3( elif alias_filter == "hibp": q = q.filter(Alias.hibp_breaches.any()) - q = q.order_by(Alias.pinned.desc()) - if sort == "old2new": q = q.order_by(Alias.created_at) elif sort == "new2old": @@ -197,6 +195,7 @@ def get_alias_infos_with_pagination_v3( ], else_=Alias.created_at, ) + q = q.order_by(Alias.pinned.desc()) q = q.order_by(latest_activity.desc()) q = list(q.limit(PAGE_LIMIT).offset(page_id * PAGE_LIMIT)) From 636879ac1a438d93be1eb1914b7d808524b43470 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 12:20:44 +0200 Subject: [PATCH 120/144] use another icon for pin --- app/dashboard/templates/dashboard/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dashboard/templates/dashboard/index.html b/app/dashboard/templates/dashboard/index.html index 7f85958f..7f0ae62a 100644 --- a/app/dashboard/templates/dashboard/index.html +++ b/app/dashboard/templates/dashboard/index.html @@ -263,7 +263,7 @@ {% endif %} {% if alias.pinned %} - {% endif %} From 5e9fb83150c1556d1e977b25aeaa96a9bdc8ebc8 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 14:08:52 +0200 Subject: [PATCH 121/144] fix typo --- app/dashboard/templates/dashboard/directory.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dashboard/templates/dashboard/directory.html b/app/dashboard/templates/dashboard/directory.html index 026a5ca1..34860a95 100644 --- a/app/dashboard/templates/dashboard/directory.html +++ b/app/dashboard/templates/dashboard/directory.html @@ -37,7 +37,7 @@ my_directory+anything@{{ FIRST_ALIAS_DOMAIN }} or
my_directory#anything@{{ FIRST_ALIAS_DOMAIN }}
- anything is any string composed of lowercase character.
+ anything is any string composed of lowercase characters.
You can find more info on directory on our blog post. From d8280af93cf1f20d2428301065a5509d930ff7f6 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 14:09:24 +0200 Subject: [PATCH 122/144] refactor: rename --- app/email_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index fcfa558e..30e417a9 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -501,11 +501,11 @@ def delete_all_headers_except(msg: Message, headers: [str]): del msg._headers[i] -def can_create_directory_for_address(address: str) -> bool: +def can_create_directory_for_address(email_address: str) -> bool: """return True if an email ends with one of the alias domains provided by SimpleLogin""" # not allow creating directory with premium domain for domain in ALIAS_DOMAINS: - if address.endswith("@" + domain): + if email_address.endswith("@" + domain): return True return False From 8024b35f1de43df6d34beb1a24cad712b29d21e4 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 14:11:59 +0200 Subject: [PATCH 123/144] refactor --- app/email_utils.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 30e417a9..ad7f99f0 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1,13 +1,12 @@ import base64 -import email import enum import os import quopri import random -import re2 as re import time import uuid from copy import deepcopy +from email import policy, message_from_bytes, message_from_string from email.header import decode_header, Header from email.message import Message from email.mime.multipart import MIMEMultipart @@ -18,6 +17,7 @@ from typing import Tuple, List, Optional, Union import arrow import dkim +import re2 as re import spf from email_validator import ( validate_email, @@ -511,9 +511,9 @@ def can_create_directory_for_address(email_address: str) -> bool: return False -def is_valid_alias_address_domain(address) -> bool: +def is_valid_alias_address_domain(email_address) -> bool: """Return whether an address domain might a domain handled by SimpleLogin""" - domain = get_email_domain_part(address) + domain = get_email_domain_part(email_address) if SLDomain.get_by(domain=domain): return True @@ -523,7 +523,7 @@ def is_valid_alias_address_domain(address) -> bool: return False -def email_can_be_used_as_mailbox(email: str) -> bool: +def email_can_be_used_as_mailbox(email_address: str) -> bool: """Return True if an email can be used as a personal email. Use the email domain as criteria. A domain can be used if it is not: - one of ALIAS_DOMAINS @@ -532,7 +532,7 @@ def email_can_be_used_as_mailbox(email: str) -> bool: - a disposable domain """ try: - domain = get_email_domain_part(email) + domain = get_email_domain_part(email_address) except EmailNotValidError: return False @@ -588,9 +588,9 @@ def get_mx_domain_list(domain) -> [str]: return [d[:-1] for _, d in priority_domains] -def personal_email_already_used(email: str) -> bool: +def personal_email_already_used(email_address: str) -> bool: """test if an email can be used as user email""" - if User.get_by(email=email): + if User.get_by(email=email_address): return True return False @@ -755,10 +755,10 @@ def copy(msg: Message) -> Message: except Exception: LOG.w("deepcopy fails, try string parsing") try: - return email.message_from_string(msg.as_string()) + return message_from_string(msg.as_string()) except (UnicodeEncodeError, KeyError, LookupError): LOG.w("as_string() fails, try bytes parsing") - return email.message_from_bytes(to_bytes(msg)) + return message_from_bytes(to_bytes(msg)) def to_bytes(msg: Message): @@ -768,11 +768,11 @@ def to_bytes(msg: Message): except UnicodeEncodeError: LOG.w("as_bytes fails with default policy, try SMTP policy") try: - return msg.as_bytes(policy=email.policy.SMTP) + return msg.as_bytes(policy=policy.SMTP) except UnicodeEncodeError: LOG.w("as_bytes fails with SMTP policy, try SMTPUTF8 policy") try: - return msg.as_bytes(policy=email.policy.SMTPUTF8) + return msg.as_bytes(policy=policy.SMTPUTF8) except UnicodeEncodeError: LOG.w("as_bytes fails with SMTPUTF8 policy, try converting to string") msg_string = msg.as_string() From 16dd35470f7e624c7152d98eae04ef0d86d19d96 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Tue, 21 Sep 2021 14:20:46 +0200 Subject: [PATCH 124/144] add more debug info --- email_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 2469ed1a..9c534c85 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1856,9 +1856,11 @@ def handle(envelope: Envelope) -> str: alias = Alias.get_by(email=rcpt_tos[0]) if alias: LOG.w( - "total number email log on %s is %s", + "total number email log on %s, %s is %s, %s", alias, + alias.user, EmailLog.query.filter(EmailLog.alias_id == alias.id).count(), + EmailLog.query.filter(EmailLog.user_id == alias.user_id).count(), ) if should_ignore_bounce(envelope.mail_from): From ff1238a56fe3cf731afcabed89ff853056dc4e52 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 22 Sep 2021 09:43:48 +0200 Subject: [PATCH 125/144] add alias note when auto creating alias via domain --- app/alias_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/alias_utils.py b/app/alias_utils.py index 04df5ff3..d58103a6 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -129,6 +129,7 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: # check if alias is custom-domain alias and if the custom-domain has catch-all enabled alias_domain = get_email_domain_part(address) custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain) + alias_note = "" if not custom_domain: return None @@ -137,6 +138,7 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: return None elif not custom_domain.catch_all and len(custom_domain.auto_create_rules) > 0: local = get_email_local_part(address) + for rule in custom_domain.auto_create_rules: rule: AutoCreateRule regex = re.compile(rule.regex) @@ -147,6 +149,7 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: rule.regex, custom_domain, ) + alias_note = f"Created by rule {rule.order} with regex {rule.regex}" mailboxes = rule.mailboxes break else: # no rule passes @@ -154,6 +157,7 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: return else: # catch-all is enabled mailboxes = custom_domain.mailboxes + alias_note = "Created by catch-all option" domain_user: User = custom_domain.user @@ -169,6 +173,7 @@ def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: custom_domain_id=custom_domain.id, automatic_creation=True, mailbox_id=mailboxes[0].id, + note=alias_note, ) db.session.flush() for i in range(1, len(mailboxes)): From 7674d8480ea5357bb12ddbe27270f224f3a59e33 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 22 Sep 2021 09:44:35 +0200 Subject: [PATCH 126/144] refactor: rename --- app/alias_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index d58103a6..b8cae527 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -43,7 +43,7 @@ def try_auto_create(address: str) -> Optional[Alias]: except EmailNotValidError: return None - alias = try_auto_create_catch_all_domain(address) + alias = try_auto_create_via_domain(address) if not alias: alias = try_auto_create_directory(address) @@ -122,8 +122,8 @@ def try_auto_create_directory(address: str) -> Optional[Alias]: return alias -def try_auto_create_catch_all_domain(address: str) -> Optional[Alias]: - """Try to create an alias with catch-all domain""" +def try_auto_create_via_domain(address: str) -> Optional[Alias]: + """Try to create an alias with catch-all or auto-create rules on custom domain""" # try to create alias on-the-fly with custom-domain catch-all feature # check if alias is custom-domain alias and if the custom-domain has catch-all enabled From 1a32b654d0cc37e8b0d8a8b1d631439029d0fc05 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 22 Sep 2021 09:45:00 +0200 Subject: [PATCH 127/144] refactor --- app/alias_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index b8cae527..c6ace16e 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -75,15 +75,15 @@ def try_auto_create_directory(address: str) -> Optional[Alias]: if not directory: return None - dir_user: User = directory.user + user: User = directory.user - if not dir_user.can_create_new_alias(): - send_cannot_create_directory_alias(dir_user, address, directory_name) + if not user.can_create_new_alias(): + send_cannot_create_directory_alias(user, address, directory_name) return None if directory.disabled: send_cannot_create_directory_alias_disabled( - dir_user, address, directory_name + user, address, directory_name ) return None @@ -112,7 +112,7 @@ def try_auto_create_directory(address: str) -> Optional[Alias]: "Alias %s was deleted before, cannot auto-create using directory %s, user %s", address, directory_name, - dir_user, + user, ) return None except IntegrityError: From 3443499ab9981e683cfab2f92378f2d151ca0fce Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 22 Sep 2021 09:45:42 +0200 Subject: [PATCH 128/144] add note for alias auto created with directory too --- app/alias_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index c6ace16e..f6b085ff 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -82,9 +82,7 @@ def try_auto_create_directory(address: str) -> Optional[Alias]: return None if directory.disabled: - send_cannot_create_directory_alias_disabled( - user, address, directory_name - ) + send_cannot_create_directory_alias_disabled(user, address, directory_name) return None try: @@ -97,6 +95,7 @@ def try_auto_create_directory(address: str) -> Optional[Alias]: user_id=directory.user_id, directory_id=directory.id, mailbox_id=mailboxes[0].id, + note=f"Created by directory {directory.name}", ) db.session.flush() for i in range(1, len(mailboxes)): From 3b16e502b3947f57af302a7c28d6dca9fcb07f15 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 22 Sep 2021 09:58:40 +0200 Subject: [PATCH 129/144] add debug info when an email is sent from reverse-alias --- app/email/status.py | 1 + email_handler.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/app/email/status.py b/app/email/status.py index b2fbfef9..82e30821 100644 --- a/app/email/status.py +++ b/app/email/status.py @@ -51,3 +51,4 @@ E522 = ( "550 SL E522 The user you are trying to contact is receiving mail " "at a rate that prevents additional messages from being delivered." ) +E523 = "550 SL E523 Unknown error" diff --git a/email_handler.py b/email_handler.py index 9c534c85..4fa9a21e 100644 --- a/email_handler.py +++ b/email_handler.py @@ -32,6 +32,7 @@ It should contain the following info: """ import argparse import email +import os import time import uuid from email import encoders @@ -82,6 +83,7 @@ from app.config import ( POSTMASTER, ALERT_HOTMAIL_COMPLAINT, ALERT_YAHOO_COMPLAINT, + TEMP_DIR, ) from app.email import status from app.email.rate_limit import rate_limited @@ -1848,6 +1850,26 @@ def handle(envelope: Envelope) -> str: ) return handle_bounce(envelope, email_log, msg) + # case where From: header is a reverse alias which should never happen + from_header = get_header_unicode(msg["From"]) + if from_header: + _, from_header_address = parse_full_address(from_header) + if is_reply_email(from_header_address): + LOG.e("email sent from a reverse alias %s", from_header_address) + # get more info for debug + contact = Contact.get_by(reply_email=from_header_address) + if contact: + LOG.d("%s %s %s", contact.user, contact.alias, contact) + + # To investigate. todo: remove + if TEMP_DIR: + file_name = str(uuid.uuid4()) + ".eml" + with open(os.path.join(TEMP_DIR, file_name), "wb") as f: + f.write(msg.as_bytes()) + + LOG.d("email saved to %s", file_name) + return status.E523 + if rate_limited(mail_from, rcpt_tos): LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos) From 2e6b012eff9661ae08453aeec7a1714d98c4af80 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Wed, 22 Sep 2021 15:33:08 +0200 Subject: [PATCH 130/144] set canonical tag for all html pages --- server.py | 1 + templates/base.html | 1 + 2 files changed, 2 insertions(+) diff --git a/server.py b/server.py index 3c7ef5a6..0010029b 100644 --- a/server.py +++ b/server.py @@ -626,6 +626,7 @@ def jinja2_filter(app): STATUS_PAGE_URL=STATUS_PAGE_URL, SUPPORT_EMAIL=SUPPORT_EMAIL, PGP_SIGNER=PGP_SIGNER, + current_url=request.url ) diff --git a/templates/base.html b/templates/base.html index 15476cce..9a161605 100644 --- a/templates/base.html +++ b/templates/base.html @@ -28,6 +28,7 @@ + {% block title %}{% endblock %} | SimpleLogin From d7d301b9c38f2c7e0b820aba75123aead20a5bb0 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Wed, 22 Sep 2021 16:03:58 +0200 Subject: [PATCH 131/144] add missing h1 --- app/auth/templates/auth/forgot_password.html | 2 +- app/auth/templates/auth/login.html | 2 +- app/auth/templates/auth/register.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/auth/templates/auth/forgot_password.html b/app/auth/templates/auth/forgot_password.html index 82daacfc..2c5c72b9 100644 --- a/app/auth/templates/auth/forgot_password.html +++ b/app/auth/templates/auth/forgot_password.html @@ -12,7 +12,7 @@ <form class="card" method="post"> {{ form.csrf_token }} <div class="card-body p-6"> - <div class="card-title">Forgot password</div> + <h1 class="card-title">Forgot password</h1> <div class="form-group"> <label class="form-label">Email address</label> diff --git a/app/auth/templates/auth/login.html b/app/auth/templates/auth/login.html index 82169c9f..b1474186 100644 --- a/app/auth/templates/auth/login.html +++ b/app/auth/templates/auth/login.html @@ -16,7 +16,7 @@ <form class="card" style="border-radius: 2%" method="post"> {{ form.csrf_token }} <div class="card-body p-6"> - <div class="card-title">Welcome back!</div> + <h1 class="card-title">Welcome back!</h1> <div class="form-group"> <label class="form-label">Email address</label> {{ form.email(class="form-control", type="email", autofocus="true") }} diff --git a/app/auth/templates/auth/register.html b/app/auth/templates/auth/register.html index b95b0d2d..8d6fb4ce 100644 --- a/app/auth/templates/auth/register.html +++ b/app/auth/templates/auth/register.html @@ -8,7 +8,7 @@ <form class="card" style="border-radius: 2%" method="post"> {{ form.csrf_token }} <div class="card-body p-6"> - <div class="card-title">Create new account</div> + <h1 class="card-title">Create new account</h1> <div class="form-group"> <label class="form-label">Email address</label> From c5425b0a73d5dd6126b09187e4d2bfd24d1851c4 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Wed, 22 Sep 2021 16:04:57 +0200 Subject: [PATCH 132/144] black --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index 0010029b..0805df39 100644 --- a/server.py +++ b/server.py @@ -626,7 +626,7 @@ def jinja2_filter(app): STATUS_PAGE_URL=STATUS_PAGE_URL, SUPPORT_EMAIL=SUPPORT_EMAIL, PGP_SIGNER=PGP_SIGNER, - current_url=request.url + current_url=request.url, ) From 7f5201effabfb8463acf4216a1ce5c0ec792efe8 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Wed, 22 Sep 2021 16:39:21 +0200 Subject: [PATCH 133/144] handle ValueError raised by parse_full_address --- email_handler.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/email_handler.py b/email_handler.py index 4fa9a21e..59e8925a 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1853,22 +1853,26 @@ def handle(envelope: Envelope) -> str: # case where From: header is a reverse alias which should never happen from_header = get_header_unicode(msg["From"]) if from_header: - _, from_header_address = parse_full_address(from_header) - if is_reply_email(from_header_address): - LOG.e("email sent from a reverse alias %s", from_header_address) - # get more info for debug - contact = Contact.get_by(reply_email=from_header_address) - if contact: - LOG.d("%s %s %s", contact.user, contact.alias, contact) + try: + _, from_header_address = parse_full_address(from_header) + except ValueError: + LOG.d("cannot parse the From header %s", from_header) + else: + if is_reply_email(from_header_address): + LOG.e("email sent from a reverse alias %s", from_header_address) + # get more info for debug + contact = Contact.get_by(reply_email=from_header_address) + if contact: + LOG.d("%s %s %s", contact.user, contact.alias, contact) - # To investigate. todo: remove - if TEMP_DIR: - file_name = str(uuid.uuid4()) + ".eml" - with open(os.path.join(TEMP_DIR, file_name), "wb") as f: - f.write(msg.as_bytes()) + # To investigate. todo: remove + if TEMP_DIR: + file_name = str(uuid.uuid4()) + ".eml" + with open(os.path.join(TEMP_DIR, file_name), "wb") as f: + f.write(msg.as_bytes()) - LOG.d("email saved to %s", file_name) - return status.E523 + LOG.d("email saved to %s", file_name) + return status.E523 if rate_limited(mail_from, rcpt_tos): LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos) From 0ed0ac9ea7f1949f4655f524fcc10e44c3d4c400 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:19:07 +0200 Subject: [PATCH 134/144] handle "text/x-python-script" in replace() --- app/email_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/email_utils.py b/app/email_utils.py index ad7f99f0..1d7c85ee 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -931,6 +931,7 @@ def replace(msg: Message, old, new) -> Message: or content_type == "text/calendar" or content_type == "text/directory" or content_type == "text/csv" + or content_type == "text/x-python-script" ): LOG.d("not applicable for %s", content_type) return msg @@ -959,7 +960,7 @@ def replace(msg: Message, old, new) -> Message: clone_msg.set_payload(new_parts) return clone_msg - LOG.e("Cannot replace text for %s", msg.get_content_type()) + LOG.w("Cannot replace text for %s", msg.get_content_type()) return msg From 486dd831cf481cfa940b56871656413eac9b6e84 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:25:07 +0200 Subject: [PATCH 135/144] fix canonical url --- server.py | 2 +- templates/base.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 0805df39..692df1e7 100644 --- a/server.py +++ b/server.py @@ -626,7 +626,7 @@ def jinja2_filter(app): STATUS_PAGE_URL=STATUS_PAGE_URL, SUPPORT_EMAIL=SUPPORT_EMAIL, PGP_SIGNER=PGP_SIGNER, - current_url=request.url, + canonical_url=f"{URL}{request.path}", ) diff --git a/templates/base.html b/templates/base.html index 9a161605..9eace899 100644 --- a/templates/base.html +++ b/templates/base.html @@ -28,7 +28,7 @@ <link rel="icon" href="/static/favicon.ico" type="image/x-icon"/> <link rel="shortcut icon" type="image/x-icon" href="/static/favicon.ico"/> - <link rel="canonical" href="{{ current_url }}"> + <link rel="canonical" href="{{ canonical_url }}"> <title> {% block title %}{% endblock %} | SimpleLogin From 469c2011aa42f46977735cda47f037e1215ab421 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Thu, 23 Sep 2021 10:30:59 +0200 Subject: [PATCH 136/144] use ~all instead of -all for SPF --- README.md | 2 +- app/dashboard/views/domain_detail.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bf0e5641..d2f53796 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Similar to DKIM, setting up SPF is highly recommended. Add a TXT record for `mydomain.com.` with the value: ``` -v=spf1 mx -all +v=spf1 mx ~all ``` What it means is only your server can send email with `@mydomain.com` domain. diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index 8773a970..3d40e686 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -42,7 +42,7 @@ def domain_detail_dns(custom_domain_id): custom_domain.ownership_txt_token = random_string(30) db.session.commit() - spf_record = f"v=spf1 include:{EMAIL_DOMAIN} -all" + spf_record = f"v=spf1 include:{EMAIL_DOMAIN} ~all" # hardcode the DKIM selector here dkim_cname = f"dkim._domainkey.{EMAIL_DOMAIN}" From b14534db2c3a9f5434ccaf3bfec21b602fe956a2 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Thu, 23 Sep 2021 19:32:06 +0200 Subject: [PATCH 137/144] not dkim sign --- app/email_utils.py | 4 ---- email_handler.py | 7 ------- 2 files changed, 11 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 1d7c85ee..b8ebd5d9 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -292,10 +292,6 @@ def send_email( msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click" ) - # add DKIM - email_domain = SUPPORT_EMAIL[SUPPORT_EMAIL.find("@") + 1 :] - add_dkim_signature(msg, email_domain) - msg_raw = to_bytes(msg) transaction = TransactionalEmail.create(email=to_email, commit=True) diff --git a/email_handler.py b/email_handler.py index 59e8925a..79532b1f 100644 --- a/email_handler.py +++ b/email_handler.py @@ -90,7 +90,6 @@ from app.email.rate_limit import rate_limited from app.email.spam import get_spam_score from app.email_utils import ( send_email, - add_dkim_signature, add_or_replace_header, delete_header, render, @@ -104,7 +103,6 @@ from app.email_utils import ( to_bytes, send_email_at_most_times, is_valid_alias_address_domain, - should_add_dkim_signature, add_header, get_header_unicode, generate_reply_email, @@ -828,8 +826,6 @@ def forward_email_to_mailbox( msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click" ) - add_dkim_signature(msg, EMAIL_DOMAIN) - LOG.d( "Forward mail from %s to %s, mail_options:%s, rcpt_options:%s ", contact.website_email, @@ -1062,9 +1058,6 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): envelope.rcpt_options, ) - if should_add_dkim_signature(alias_domain): - add_dkim_signature(msg, alias_domain) - # generate a mail_from for VERP verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}" From 51a7dbfa52ec6371f84c71b97f19432c0da2b6ff Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Sat, 25 Sep 2021 18:47:15 +0200 Subject: [PATCH 138/144] Revert "not dkim sign" This reverts commit b14534db2c3a9f5434ccaf3bfec21b602fe956a2. --- app/email_utils.py | 4 ++++ email_handler.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/app/email_utils.py b/app/email_utils.py index b8ebd5d9..1d7c85ee 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -292,6 +292,10 @@ def send_email( msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click" ) + # add DKIM + email_domain = SUPPORT_EMAIL[SUPPORT_EMAIL.find("@") + 1 :] + add_dkim_signature(msg, email_domain) + msg_raw = to_bytes(msg) transaction = TransactionalEmail.create(email=to_email, commit=True) diff --git a/email_handler.py b/email_handler.py index 79532b1f..59e8925a 100644 --- a/email_handler.py +++ b/email_handler.py @@ -90,6 +90,7 @@ from app.email.rate_limit import rate_limited from app.email.spam import get_spam_score from app.email_utils import ( send_email, + add_dkim_signature, add_or_replace_header, delete_header, render, @@ -103,6 +104,7 @@ from app.email_utils import ( to_bytes, send_email_at_most_times, is_valid_alias_address_domain, + should_add_dkim_signature, add_header, get_header_unicode, generate_reply_email, @@ -826,6 +828,8 @@ def forward_email_to_mailbox( msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click" ) + add_dkim_signature(msg, EMAIL_DOMAIN) + LOG.d( "Forward mail from %s to %s, mail_options:%s, rcpt_options:%s ", contact.website_email, @@ -1058,6 +1062,9 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): envelope.rcpt_options, ) + if should_add_dkim_signature(alias_domain): + add_dkim_signature(msg, alias_domain) + # generate a mail_from for VERP verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}" From 408322217d734ac222c9a6a9974985040e647c6b Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Mon, 27 Sep 2021 09:59:33 +0200 Subject: [PATCH 139/144] allow user having apple subscription to switch to web subscription --- app/dashboard/templates/dashboard/setting.html | 16 ++++++++++++---- app/dashboard/views/pricing.py | 5 +++++ app/models.py | 4 ---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/app/dashboard/templates/dashboard/setting.html b/app/dashboard/templates/dashboard/setting.html index e3696bf0..e3909e62 100644 --- a/app/dashboard/templates/dashboard/setting.html +++ b/app/dashboard/templates/dashboard/setting.html @@ -49,8 +49,15 @@ {% endif %} {% elif apple_sub and apple_sub.is_valid() %} - You are on the Premium plan which expires {{ apple_sub.expires_date | dt }} + You are on the Premium plan (subscribed via Apple) which expires {{ apple_sub.expires_date | dt }} ({{ apple_sub.expires_date.format("YYYY-MM-DD") }}). + <div class="alert alert-info"> + If you want to subscribe via the Web instead, please make sure to cancel your subscription + on Apple first. + <a href="{{ url_for('dashboard.pricing') }}" + class="">Upgrade <i class="fa fa-arrow-right" aria-hidden="true"></i></a> + </div> + {% elif coinbase_sub and coinbase_sub.is_active() %} You are on the Premium plan which expires {{ coinbase_sub.end_at | dt }} ({{ coinbase_sub.end_at.format("YYYY-MM-DD") }}). @@ -252,7 +259,7 @@ <form method="post" action="#random-alias-suffix" class="form-inline"> <input type="hidden" name="form-name" value="random-alias-suffix"> <select class="form-control mr-sm-2" name="random-alias-suffix-generator"> - + <option value="0" {% if current_user.random_alias_suffix==0 %} selected {% endif %}> Random word from our dictionary </option> @@ -343,8 +350,9 @@ <input type="hidden" name="form-name" value="sender-in-ra"> <div class="form-check"> <input type="checkbox" id="include-sender-ra" name="enable" -{# todo: remove current_user.include_sender_in_reverse_alias is none condition #} - {% if current_user.include_sender_in_reverse_alias is none or current_user.include_sender_in_reverse_alias %} checked {% endif %} class="form-check-input"> + {# todo: remove current_user.include_sender_in_reverse_alias is none condition #} + {% if current_user.include_sender_in_reverse_alias is none or current_user.include_sender_in_reverse_alias %} + checked {% endif %} class="form-check-input"> <label for="include-sender-ra">Include sender address in reverse-alias</label> </div> <button type="submit" class="btn btn-outline-primary">Update</button> diff --git a/app/dashboard/views/pricing.py b/app/dashboard/views/pricing.py index 7378d18d..da16424a 100644 --- a/app/dashboard/views/pricing.py +++ b/app/dashboard/views/pricing.py @@ -12,6 +12,7 @@ from app.config import ( ) from app.dashboard.base import dashboard_bp from app.log import LOG +from app.models import AppleSubscription @dashboard_bp.route("/pricing", methods=["GET", "POST"]) @@ -21,6 +22,10 @@ def pricing(): flash("You are already a premium user", "warning") return redirect(url_for("dashboard.index")) + apple_sub: AppleSubscription = AppleSubscription.get_by(user_id=current_user.id) + if apple_sub and apple_sub.is_valid(): + flash("Please make sure to cancel your subscription on Apple first", "warning") + return render_template( "dashboard/pricing.html", PADDLE_VENDOR_ID=PADDLE_VENDOR_ID, diff --git a/app/models.py b/app/models.py index 8983e72d..3e7f1895 100644 --- a/app/models.py +++ b/app/models.py @@ -477,10 +477,6 @@ class User(db.Model, ModelMixin, UserMixin, PasswordOracle): if sub and not sub.cancelled: return False - apple_sub: AppleSubscription = AppleSubscription.get_by(user_id=self.id) - if apple_sub and apple_sub.is_valid(): - return False - manual_sub: ManualSubscription = ManualSubscription.get_by(user_id=self.id) # user who has giveaway premium can decide to upgrade if manual_sub and manual_sub.is_active() and not manual_sub.is_giveaway: From 3ad961bfb9f158f54c5f63e36983736930385342 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:21:49 +0200 Subject: [PATCH 140/144] ignore contact name that has hex ascii code \x00 --- email_handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/email_handler.py b/email_handler.py index 59e8925a..b234ce52 100644 --- a/email_handler.py +++ b/email_handler.py @@ -211,6 +211,10 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con contact_email = sanitize_email(contact_email) + if contact_name and "\x00" in contact_name: + LOG.w("issue with contact name %s", contact_name) + contact_name = "" + contact = Contact.get_by(alias_id=alias.id, website_email=contact_email) if contact: if contact.name != contact_name: From 8301015afde9372dcf6bc7764175c49b99a93461 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Mon, 27 Sep 2021 12:13:41 +0200 Subject: [PATCH 141/144] do not use email_validator in get_email_domain_part() --- app/alias_utils.py | 1 - app/email_utils.py | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/alias_utils.py b/app/alias_utils.py index f6b085ff..a5f1a7d5 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -128,7 +128,6 @@ def try_auto_create_via_domain(address: str) -> Optional[Alias]: # check if alias is custom-domain alias and if the custom-domain has catch-all enabled alias_domain = get_email_domain_part(address) custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain) - alias_note = "" if not custom_domain: return None diff --git a/app/email_utils.py b/app/email_utils.py index 1d7c85ee..cdcf986e 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -404,10 +404,8 @@ def get_email_domain_part(address): Get the domain part from email ab@cd.com -> cd.com """ - r: ValidatedEmail = validate_email( - address, check_deliverability=False, allow_smtputf8=False - ) - return r.domain + address = sanitize_email(address) + return address[address.find("@") + 1 :] # headers used to DKIM sign in order of preference @@ -532,7 +530,9 @@ def email_can_be_used_as_mailbox(email_address: str) -> bool: - a disposable domain """ try: - domain = get_email_domain_part(email_address) + domain = validate_email( + email_address, check_deliverability=False, allow_smtputf8=False + ).domain except EmailNotValidError: return False From 339d611e63daea390be066d55522ff31b2eec73c Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Mon, 27 Sep 2021 12:19:33 +0200 Subject: [PATCH 142/144] remove Contact.from_header column --- app/models.py | 2 -- email_handler.py | 11 ------- .../versions/2021_092712_5ee767807344_.py | 29 +++++++++++++++++++ 3 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 migrations/versions/2021_092712_5ee767807344_.py diff --git a/app/models.py b/app/models.py index 3e7f1895..8488ed21 100644 --- a/app/models.py +++ b/app/models.py @@ -1417,8 +1417,6 @@ class Contact(db.Model, ModelMixin): # to investigate why the website_email is sometimes not correctly parsed # the envelope mail_from mail_from = db.Column(db.Text, nullable=True, default=None) - # the message["From"] header - from_header = db.Column(db.Text, nullable=True, default=None) # a contact can have an empty email address, in this case it can't receive emails invalid_email = db.Column( diff --git a/email_handler.py b/email_handler.py index b234ce52..99803838 100644 --- a/email_handler.py +++ b/email_handler.py @@ -237,16 +237,6 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con ) contact.mail_from = mail_from db.session.commit() - - if not contact.from_header and from_header: - LOG.d( - "Set contact from_header %s: %s to %s", - contact, - contact.from_header, - from_header, - ) - contact.from_header = from_header - db.session.commit() else: LOG.d( "create contact %s for alias %s", @@ -261,7 +251,6 @@ def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Con website_email=contact_email, name=contact_name, mail_from=mail_from, - from_header=from_header, reply_email=generate_reply_email(contact_email, alias.user) if is_valid_email(contact_email) else NOREPLY, diff --git a/migrations/versions/2021_092712_5ee767807344_.py b/migrations/versions/2021_092712_5ee767807344_.py new file mode 100644 index 00000000..ad5a6430 --- /dev/null +++ b/migrations/versions/2021_092712_5ee767807344_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: 5ee767807344 +Revises: b8b4f9598240 +Create Date: 2021-09-27 12:19:14.398709 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '5ee767807344' +down_revision = 'b8b4f9598240' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('contact', 'from_header') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('contact', sa.Column('from_header', sa.TEXT(), autoincrement=False, nullable=True)) + # ### end Alembic commands ### From e3f81bc4e42d2c0226cc5375807a94ab1b9b18fc Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Mon, 27 Sep 2021 15:51:09 +0200 Subject: [PATCH 143/144] fix subscription reminder sent to lifetime user --- cron.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cron.py b/cron.py index 0a594d14..9bbef57d 100644 --- a/cron.py +++ b/cron.py @@ -123,6 +123,10 @@ def notify_premium_end(): >= arrow.now().shift(days=2).date() ): user = sub.user + + if user.lifetime: + continue + LOG.d(f"Send subscription ending soon email to user {user}") send_email( From 9528bdcb2e6c65ffad140de3685ae56e472d40ee Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim <nguyenkims@users.noreply.github.com> Date: Mon, 27 Sep 2021 15:58:04 +0200 Subject: [PATCH 144/144] user can enable the ignore_loop_email --- .../templates/dashboard/setting.html | 23 +++++++++++++++++++ app/dashboard/views/setting.py | 9 ++++++++ 2 files changed, 32 insertions(+) diff --git a/app/dashboard/templates/dashboard/setting.html b/app/dashboard/templates/dashboard/setting.html index e3909e62..5dd5bba6 100644 --- a/app/dashboard/templates/dashboard/setting.html +++ b/app/dashboard/templates/dashboard/setting.html @@ -382,6 +382,29 @@ </div> <!-- END Always expand alias info --> + <!-- Ignore Loop Email --> + <div class="card" id="ignore-loop-email-section"> + <div class="card-body"> + <div class="card-title">Ignore Loop Emails</div> + <div class="mb-3"> + On some email clients, "Reply All" automatically includes your alias that + would send the same email to your mailbox. + <br> + You can disable these "loop" emails by enabling this option. + </div> + <form method="post" action="#ignore-loop-email-section"> + <input type="hidden" name="form-name" value="ignore-loop-email"> + <div class="form-check"> + <input type="checkbox" id="ignore-loop-email" name="enable" + {% if current_user.ignore_loop_email %} checked {% endif %} class="form-check-input"> + <label for="ignore-loop-email">Ignore Loop Emails</label> + </div> + <button type="submit" class="btn btn-outline-primary">Update</button> + </form> + </div> + </div> + <!-- END Ignore Loop Email --> + <div class="card"> <div class="card-body"> <div class="card-title">Quarantine</div> diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index 11b2a9cd..873556af 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -300,6 +300,15 @@ def setting(): db.session.commit() flash("Your preference has been updated", "success") return redirect(url_for("dashboard.setting")) + elif request.form.get("form-name") == "ignore-loop-email": + choose = request.form.get("enable") + if choose == "on": + current_user.ignore_loop_email = True + else: + current_user.ignore_loop_email = False + db.session.commit() + flash("Your preference has been updated", "success") + return redirect(url_for("dashboard.setting")) elif request.form.get("form-name") == "export-data": return redirect(url_for("api.export_data"))