fix: monitoring table name (#2120)

* fix: commit transaction after taking event

* feat: allow to reconnect to postgres for event listener

* chore: log sync events pending to process to metrics

* fix: make dead_letter runner able to process events without needing to have lock on the event

* chore: close Session after reconnect

* refactor: make EventSource emit only events that can be processed

.dockerignore

@@ -6,5 +6,12 @@ db.sqlite
 .vscode
 .DS_Store
 config
-LICENSE
-README.md
+adhoc
+static/node_modules
+db.sqlite-journal
+static/upload
+venv/
+.venv
+.coverage
+htmlcov
+.git/

.flake8

@@ -0,0 +1,26 @@
+[flake8]
+max-line-length = 88
+select = C,E,F,W,B,B902,B903,B904,B950
+extend-ignore =
+    # For black compatibility
+    E203,
+    E501,
+    # Ignore "f-string is missing placeholders"
+    F541,
+    # allow bare except
+    E722, B001
+exclude =
+    .git,
+    __pycache__,
+    .pytest_cache,
+    .venv,
+    static,
+    templates,
+    # migrations are generated by alembic
+    migrations,
+    docs,
+    shell.py
+
+per-file-ignores =
+    # ignore unused imports in __init__
+    __init__.py:F401

.gitattributes

@@ -0,0 +1,3 @@
+# https://github.com/github/linguist#overrides
+static/* linguist-vendored
+docs/* linguist-documentation

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+## code changes will send PR to following users
+* @acasajus @cquintana92 @nguyenkims

.github/FUNDING.yml

@@ -1 +1 @@
-patreon: simplelogin
+open_collective: simplelogin

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,39 @@
+---
+name: Bug report
+about: Create a report to help us improve SimpleLogin.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Please note that this is only for bug report.
+
+For help on your account, please reach out to us at hi[at]simplelogin.io. Please make sure to check out [our FAQ](https://simplelogin.io/faq/) that contains  frequently asked questions.
+
+
+For feature request, you can use our [forum](https://github.com/simple-login/app/discussions/categories/feature-request).
+
+For self-hosted question/issue, please ask in [self-hosted forum](https://github.com/simple-login/app/discussions/categories/self-hosting-question)
+
+## Prerequisites
+- [ ] I have searched open and closed issues to make sure that the bug has not yet been reported.
+
+## Bug report
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (If applicable):**
+ - OS: Linux, Mac, Windows
+ - Browser: Firefox, Chrome, Brave, Safari
+ - Version [e.g. 78]
+
+**Additional context**
+Add any other context about the problem here.

.github/changelog_configuration.json

@@ -0,0 +1,23 @@
+{
+  "template": "${{CHANGELOG}}\n\n<details>\n<summary>Uncategorized</summary>\n\n${{UNCATEGORIZED}}\n</details>",
+  "pr_template": "- ${{TITLE}} #${{NUMBER}}",
+  "empty_template": "- no changes",
+  "categories": [
+    {
+      "title": "## 🚀 Features",
+      "labels": ["feature"]
+    },
+    {
+      "title": "## 🐛 Fixes",
+      "labels": ["fix", "bug"]
+    },
+    {
+      "title": "## 🔧 Enhancements",
+      "labels": ["enhancement"]
+    }
+  ],
+  "ignore_labels": ["ignore"],
+  "tag_resolver": {
+    "method": "semver"
+  }
+}

.github/workflows/main.yml

@@ -1,48 +1,244 @@
-name: Run tests & Public to Docker Registry
+name: Test and lint
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  build:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
 
+      - name: Install poetry
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'poetry'
+
+      - name: Install OS dependencies
+        if: ${{ matrix.python-version }} == '3.10'
+        run: |
+          sudo apt update
+          sudo apt install -y libre2-dev libpq-dev
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction
+
+      - name: Check formatting & linting
+        run: |
+          poetry run pre-commit run --all-files
+
+
+  test:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.7]
+        python-version: ["3.10"]
+
+    # service containers to run with `postgres-job`
+    services:
+      # label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:13
+        # service environment variables
+        # `POSTGRES_HOST` is `postgres`
+        env:
+          # optional (defaults to `postgres`)
+          POSTGRES_DB: test
+          # required
+          POSTGRES_PASSWORD: test
+          # optional (defaults to `5432`)
+          POSTGRES_PORT: 5432
+          # optional (defaults to `postgres`)
+          POSTGRES_USER: test
+        ports:
+          - 15432:5432
+        # set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
+      - name: Check out repo
+        uses: actions/checkout@v3
 
-    - name: Test formatting
-      run: |
-        pip install black
-        black --check .
+      - name: Install poetry
+        run: pipx install poetry
 
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
 
-    - name: Test with pytest
-      run: |
-        pip install pytest
-        pytest
+      - name: Install OS dependencies
+        if: ${{ matrix.python-version }} == '3.10'
+        run: |
+          sudo apt update
+          sudo apt install -y libre2-dev libpq-dev
 
-    - name: Publish to Docker Registry
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name: simplelogin/app-ci
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction
 
-    - name: Send Telegram message
-      uses: appleboy/telegram-action@master
-      with:
-        to: ${{ secrets.TELEGRAM_TO }}
-        token: ${{ secrets.TELEGRAM_TOKEN }}
-        args: Docker image pushed on ${{ github.ref }}
+
+      - name: Start Redis v6
+        uses: superchargejs/redis-github-action@1.1.0
+        with:
+          redis-version: 6
+
+      - name: Run db migration
+        run: |
+          CONFIG=tests/test.env poetry run alembic upgrade head
+
+      - name: Prepare version file
+        run: |
+          scripts/generate-build-info.sh ${{ github.sha }}
+          cat app/build_info.py
+
+      - name: Test with pytest
+        run: |
+          poetry run pytest
+        env:
+          GITHUB_ACTIONS_TEST: true
+
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@v2
+        with:
+          name: code-coverage-report
+          path: htmlcov
+
+  build:
+    runs-on: ubuntu-latest
+    needs: ['test', 'lint']
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v'))
+
+    steps:
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: simplelogin/app-ci
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # We need to checkout the repository in order for the "Create Sentry release" to work
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Create Sentry release
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+        with:
+          ignore_missing: true
+          ignore_empty: true
+
+      - name: Prepare version file
+        run: |
+          scripts/generate-build-info.sh ${{ github.sha }}
+          cat app/build_info.py
+
+      - name: Build image and publish to Docker Registry
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+
+
+      #- name: Send Telegram message
+      #  uses: appleboy/telegram-action@master
+      #  with:
+      #    to: ${{ secrets.TELEGRAM_TO }}
+      #    token: ${{ secrets.TELEGRAM_TOKEN }}
+      #    args: Docker image pushed on ${{ github.ref }}
+
+      # If we have generated a tag, generate the changelog, send a notification to slack and create the GitHub release
+      - name: Build Changelog
+        id: build_changelog
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: mikepenz/release-changelog-builder-action@v3
+        with:
+          configuration: ".github/changelog_configuration.json"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Prepare Slack notification contents
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          changelog=$(cat << EOH
+          ${{ steps.build_changelog.outputs.changelog }}
+          EOH
+          )
+          messageWithoutNewlines=$(echo "${changelog}" | awk '{printf "%s\\n", $0}')
+          messageWithoutDoubleQuotes=$(echo "${messageWithoutNewlines}" | sed "s/\"/'/g")
+          echo "${messageWithoutDoubleQuotes}"
+
+          echo "SLACK_CHANGELOG=${messageWithoutDoubleQuotes}" >> $GITHUB_ENV
+
+      - name: Post notification to Slack
+        uses: slackapi/slack-github-action@v1.19.0
+        if: startsWith(github.ref, 'refs/tags/v')
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "New tag created",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "text": {
+                      "type": "mrkdwn",
+                      "text": "*Tag: ${{ github.ref_name }}* (${{ github.sha }})"
+                    }
+                },
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "*Changelog:*\n${{ env.SLACK_CHANGELOG }}"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+      - name: Create GitHub Release
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: actions/create-release@v1
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          body: ${{ steps.build_changelog.outputs.changelog }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -7,4 +7,12 @@ db.sqlite
 .DS_Store
 config
 static/node_modules
-db.sqlite-journal
+db.sqlite-journal
+static/upload
+venv/
+.venv
+.python-version
+.coverage
+htmlcov
+adhoc
+.env.*

.jshintrc

@@ -0,0 +1,3 @@
+{
+  "esversion": 8
+}

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+exclude: "(migrations|static/node_modules|static/assets|static/vendor)"
+default_language_version:
+  python: python3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+      - id: check-yaml
+      - id: trailing-whitespace
+  - repo: https://github.com/Riverside-Healthcare/djLint
+    rev: v1.3.0
+    hooks:
+      - id: djlint-jinja
+        files: '.*\.html'
+        entry: djlint --reformat
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.1.5
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+

.pylintrc

@@ -0,0 +1,227 @@
+[MASTER]
+extension-pkg-allow-list=re2
+
+fail-under=7.0
+ignore=CVS
+ignore-paths=migrations
+ignore-patterns=^\.#
+jobs=0
+
+[MESSAGES CONTROL]
+disable=missing-function-docstring,
+        missing-module-docstring,
+        duplicate-code,
+        #import-error,
+        missing-class-docstring,
+        useless-object-inheritance,
+        use-dict-literal,
+        logging-format-interpolation,
+        consider-using-f-string,
+        unnecessary-comprehension,
+        inconsistent-return-statements,
+        wrong-import-order,
+        line-too-long,
+        invalid-name,
+        global-statement,
+        no-else-return,
+        unspecified-encoding,
+        logging-fstring-interpolation,
+        too-few-public-methods,
+        bare-except,
+        fixme,
+        unnecessary-pass,
+        f-string-without-interpolation,
+        super-init-not-called,
+        unused-argument,
+        ungrouped-imports,
+        too-many-locals,
+        consider-using-with,
+        too-many-statements,
+        consider-using-set-comprehension,
+        unidiomatic-typecheck,
+        useless-else-on-loop,
+        too-many-return-statements,
+        broad-except,
+        protected-access,
+        consider-using-enumerate,
+        too-many-nested-blocks,
+        too-many-branches,
+        simplifiable-if-expression,
+        possibly-unused-variable,
+        pointless-string-statement,
+        wrong-import-position,
+        redefined-outer-name,
+        raise-missing-from,
+        logging-too-few-args,
+        redefined-builtin,
+        too-many-arguments,
+        import-outside-toplevel,
+        redefined-argument-from-local,
+        logging-too-many-args,
+        too-many-instance-attributes,
+        unreachable,
+        no-name-in-module,
+        no-member,
+        consider-using-ternary,
+        too-many-lines,
+        arguments-differ,
+        too-many-public-methods,
+        unused-variable,
+        consider-using-dict-items,
+        consider-using-in,
+        reimported,
+        too-many-boolean-expressions,
+        cyclic-import,
+        not-callable, # (paddle_utils.py) verifier.verify cannot be called (although it can)
+        abstract-method, # (models.py)
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style. If left empty, argument names will be checked with the set
+# naming style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style. If left empty, attribute names will be checked with the set naming
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style. If left empty, class attribute names will be checked
+# with the set naming style.
+#class-attribute-rgx=
+
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
+
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style. If left empty, class constant names will be checked with
+# the set naming style.
+#class-const-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style. If left empty, class names will be checked with the set naming style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style. If left empty, constant names will be checked with the set naming
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style. If left empty, function names will be checked with the set
+# naming style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style. If left empty, inline iteration names will be checked
+# with the set naming style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style. If left empty, method names will be checked with the set naming style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style. If left empty, module names will be checked with the set naming style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Regular expression matching correct type variable names. If left empty, type
+# variable names will be checked with the set naming style.
+#typevar-rgx=
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style. If left empty, variable names will be checked with the set
+# naming style.
+#variable-rgx=
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[FORMAT]
+max-line-length=88
+single-line-if-stmt=yes

.version

@@ -0,0 +1 @@
+dev

CHANGELOG

@@ -6,7 +6,94 @@ The version corresponds to SimpleLogin Docker `image tag`.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [3.4.0] - 2021-04-06
+Support ARM arch
+Remove unused config like DEBUG, CLOUDWATCH, DKIM_PUBLIC_KEY_PATH, DKIM_DNS_VALUE
+Handle auto responder email
+Inform user when their alias has been transferred to another user
+Use alias transfer_token
+Improve logging
+Add /api/export/data, /api/export/aliases endpoints
+Take into account mailbox when importing/exporting aliases
+Multiple bug fixes
+Code refactoring
+Add ENABLE_SPAM_ASSASSIN config
+
+## [3.3.0] - 2021-03-05
+Notify user when reply cannot be sent
+User can choose default domain for random alias
+enable LOCAL_FILE_UPLOAD by default
+fix user has to login again after quitting the browser
+login user in api auth endpoints
+Create POST /api/api_key
+Add GET /api/logout
+Add setup-done page
+Add PublicDomain
+User can choose a random alias domain in a list of public domains
+User can choose mailboxes for a domain
+Return support_pgp in GET /api/v2/aliases
+Self hosting improvements
+Improve Search
+Use poetry instead of pip
+Add PATCH /api/user_info
+Add GET /api/setting
+Add GET /api/setting/domains
+Add PATCH /api/setting
+Add "Generic Subject" option
+Add /v2/setting/domains
+Add /api/v5/alias/options
+Add GET /api/custom_domains
+Add GET /api/custom_domains/:custom_domain_id/trash
+Able to disable a directory
+Use VERP: send email from bounce address
+Use VERP for transactional email: remove SENDER, SENDER_DIR
+Use "John Wick - john at wick.com" as default sender format
+Able to transfer an alias
+
+## [3.2.2] - 2020-06-15
+Fix POST /v2/alias/custom/new when DISABLE_ALIAS_SUFFIX is set
+
+## [3.2.1] - 2020-06-15
+Fix regressions introduced in 3.2.0 regarding DISABLE_ALIAS_SUFFIX option
+
+## [3.2.0] - 2020-06-10
+Make FIDO available
+Fix "remove the reverse-alias" when replying
+Update GET /mailboxes
+Create POST /api/v3/alias/custom/new
+Add PGP for contact
+
+## [3.1.1] - 2020-05-27
+Fix alias creation
+
+## [3.1.0] - 2020-05-09
+Remove social login signup
+More simple UI with advanced options hidden by default
+Use pagination for alias page
+Use Ajax for alias note and mailbox update
+Alias can have a name
+Global stats
+DMARC support for custom domain
+Enforce SPF
+FIDO support (beta)
+Able to disable onboarding emails
+
+
+## [3.0.1] - 2020-04-13
+Fix compatibility with 2x version
+Fix "Content-Transfer-Encoding" issue https://github.com/simple-login/app/issues/125
+
+## [3.0.0] - 2020-04-13
+New endpoints to create/update aliases:
+    PUT /api/aliases/:alias_id
+    GET /api/aliases/:alias_id/contacts
+    POST /api/aliases/:alias_id/contacts
+    GET /api/v2/aliases
+(Optional) Spam detection by Spamassassin
+Handling for bounced emails
+Support Multiple recipients (in To and Cc headers)
+
+## [2.1.0] - 2020-03-23
 Support PGP
 
 ## [2.0.0] - 2020-03-13
@@ -30,7 +117,7 @@ Add SUPPORT_NAME param to set a support email name.
 
 ## [1.0.1] - 2020-01-28
 
-Simplify config file. 
+Simplify config file.
 
 ## [1.0.0] - 2020-01-22
 

CONTRIBUTING.md

@@ -0,0 +1,228 @@
+Thanks for taking the time to contribute! 🎉👍
+
+Before working on a new feature, please get in touch with us at dev[at]simplelogin.io to avoid duplication.
+We can also discuss the best way to implement it.
+
+The project uses Flask, Python3.7+ and requires Postgres 12+ as dependency.
+
+## General Architecture
+
+<p align="center">
+    <img src="./docs/archi.png" height="450px">
+</p>
+
+SimpleLogin backend consists of 2 main components:
+
+- the `webapp` used by several clients: the web app, the browser extensions (Chrome & Firefox for now), OAuth clients (apps that integrate "Sign in with SimpleLogin" button) and mobile apps.
+
+- the `email handler`: implements the email forwarding (i.e. alias receiving email) and email sending (i.e. alias sending email).
+
+## Install dependencies
+
+The project requires:
+- Python 3.7+ and [poetry](https://python-poetry.org/) to manage dependencies
+- Node v10 for front-end.
+- Postgres 12+
+
+First, install all dependencies by running the following command.
+Feel free to use `virtualenv` or similar tools to isolate development environment.
+
+```bash
+poetry install
+```
+
+On Mac, sometimes you might need to install some other packages via `brew`:
+
+```bash
+brew install pkg-config libffi openssl postgresql@13
+```
+
+You also need to install `gpg` tool, on Mac it can be done with:
+
+```bash
+brew install gnupg
+```
+
+If you see the `pyre2` package in the error message, you might need to install its dependencies with `brew`.
+More info on https://github.com/andreasvc/pyre2
+
+```bash
+brew install -s re2 pybind11
+```
+
+## Linting and static analysis
+
+We use pre-commit to run all our linting and static analysis checks. Please run
+
+```bash
+poetry run pre-commit install
+```
+
+To install it in your development environment.
+
+## Run tests
+
+For most tests, you will need to have ``redis`` installed and started on your machine (listening on port 6379).
+
+```bash
+sh scripts/run-test.sh
+```
+
+You can also run tests using a local Postgres DB to speed things up. This can be done by
+
+- creating an empty test DB and running the database migration by `dropdb test && createdb test && DB_URI=postgresql://localhost:5432/test alembic upgrade head`
+
+- replacing the `DB_URI` in `test.env` file by `DB_URI=postgresql://localhost:5432/test`
+
+## Run the code locally
+
+Install npm packages
+
+```bash
+cd static && npm install
+```
+
+To run the code locally, please create a local setting file based on `example.env`:
+
+```
+cp example.env .env
+```
+
+You need to edit your .env to reflect the postgres exposed port, edit the `DB_URI` to:
+
+```
+DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin
+```
+
+Run the postgres database:
+
+```bash
+docker run -e POSTGRES_PASSWORD=mypassword -e POSTGRES_USER=myuser -e POSTGRES_DB=simplelogin -p 15432:5432 postgres:13
+```
+
+To run the server:
+
+```
+alembic upgrade head && flask dummy-data && python3 server.py
+```
+
+then open http://localhost:7777, you should be able to login with `john@wick.com / password` account.
+
+You might need to change the `.env` file for developing certain features. This file is ignored by git.
+
+## Database migration
+
+The database migration is handled by `alembic`
+
+Whenever the model changes, a new migration has to be created.
+
+If you have Docker installed, you can create the migration by the following script:
+
+```bash
+sh scripts/new-migration.sh
+```
+
+Make sure to review the migration script before committing it.
+Sometimes (very rarely though), the automatically generated script can be incorrect.
+
+We cannot use the local database to generate migration script as the local database doesn't use migration.
+It is created via `db.create_all()` (cf `fake_data()` method). This is convenient for development and
+unit tests as we don't have to wait for the migration.
+
+## Reset database
+
+There are two scripts to reset your local db to an empty state:
+
+- `scripts/reset_local_db.sh` will reset your development db to the latest migration version and add the development data needed to run the
+server.py locally.
+- `scripts/reset_test_db.sh` will reset your test db to the latest migration without adding the dev server data to prevent interferring with
+the tests.
+
+## Code structure
+
+The repo consists of the three following entry points:
+
+- wsgi.py and server.py: the webapp.
+- email_handler.py: the email handler.
+- cron.py: the cronjob.
+
+Here are the small sum-ups of the directory structures and their roles:
+
+- app/: main Flask app. It is structured into different packages representing different features like oauth,  api, dashboard, etc.
+- local_data/: contains files to facilitate the local development. They are replaced during the deployment.
+- migrations/: generated by flask-migrate. Edit these files will be only edited when you spot (very rare) errors on the database migration files.
+- static/: files available at `/static` url.
+- templates/: contains both html and email templates.
+- tests/: tests. We don't really distinguish unit, functional or integration test. A test is simply here to make sure a feature works correctly.
+
+## Pull request
+
+The code is formatted using [ruff](https://github.com/astral-sh/ruff), to format the code, simply run
+
+```
+poetry run ruff format .
+```
+
+The code is also checked with `flake8`, make sure to run `flake8` before creating the pull request by
+
+```bash
+poetry run flake8
+```
+
+For HTML templates, we use `djlint`. Before creating a pull request, please run
+
+```bash
+poetry run djlint --check templates
+```
+
+If some files aren't properly formatted, you can format all files with
+
+```bash
+poetry run djlint --reformat .
+```
+
+## Test sending email
+
+[swaks](http://www.jetmore.org/john/code/swaks/) is used for sending test emails to the `email_handler`.
+
+[mailcatcher](https://github.com/sj26/mailcatcher) or [MailHog](https://github.com/mailhog/MailHog) can be used as a MTA to receive emails.
+
+Here's how set up the email handler:
+
+1) run mailcatcher or MailHog
+
+```bash
+mailcatcher
+```
+
+2) Make sure to set the following variables in the `.env` file
+
+```
+# comment out this variable
+# NOT_SEND_EMAIL=true
+
+# So the emails will be sent to mailcatcher/MailHog
+POSTFIX_SERVER=localhost
+POSTFIX_PORT=1025
+```
+
+3) Run email_handler
+
+```bash
+python email_handler.py
+```
+
+4) Send a test email
+
+```bash
+swaks --to e1@sl.local --from hey@google.com --server 127.0.0.1:20381
+```
+
+Now open http://localhost:1080/ (or http://localhost:1080/ for MailHog), you should see the forwarded email.
+
+## Job runner
+
+Some features require a job handler (such as GDPR data export). To test such feature you need to run the job_runner
+```bash
+python job_runner.py
+```

Dockerfile

@@ -2,15 +2,38 @@
 FROM node:10.17.0-alpine AS npm
 WORKDIR /code
 COPY ./static/package*.json /code/static/
-RUN cd /code/static && npm install
+RUN cd /code/static && npm ci
 
+# Main image
+FROM python:3.10
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE 1
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED 1
+
+# Add poetry to PATH
+ENV PATH="${PATH}:/root/.local/bin"
 
-FROM python:3.7
 WORKDIR /code
 
-# install dependencies
-COPY ./requirements.txt ./
-RUN pip3 install --no-cache-dir -r requirements.txt
+# Copy poetry files
+COPY poetry.lock pyproject.toml ./
+
+# Install and setup poetry
+RUN pip install -U pip \
+    && apt-get update \
+    && apt install -y curl netcat-traditional gcc python3-dev gnupg git libre2-dev cmake ninja-build\
+    && curl -sSL https://install.python-poetry.org | python3 - \
+    # Remove curl and netcat from the image
+    && apt-get purge -y curl netcat-traditional \
+    # Run poetry
+    && poetry config virtualenvs.create false \
+    && poetry install  --no-interaction --no-ansi --no-root \
+    # Clear apt cache \
+    && apt-get purge -y libre2-dev cmake ninja-build\
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # copy npm packages
 COPY --from=npm /code /code

LICENSE

@@ -1,21 +1,661 @@
-MIT License
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
 
-Copyright (c) 2020 SimpleLogin
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+                            Preamble
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

README.md

@@ -1,96 +1,42 @@
-<p align="center">
-    <a href="https://simplelogin.io">
-        <img src="./docs/diagram.png" height="300px">
-    </a>
-</p>
 
-[SimpleLogin](https://simplelogin.io) | Privacy-First Email Forwarding and Identity Provider Service
+
+[SimpleLogin](https://simplelogin.io) | Protect your online identity with email alias
 ---
 <p>
-<a href="https://chrome.google.com/webstore/detail/simplelogin-protect-your/dphilobhebphkdjbpfohgikllaljmgbn">
+<a href="https://chrome.google.com/webstore/detail/dphilobhebphkdjbpfohgikllaljmgbn">
     <img src="https://img.shields.io/chrome-web-store/rating/dphilobhebphkdjbpfohgikllaljmgbn?label=Chrome%20Extension">
 </a>
 
-<a href="https://addons.mozilla.org/en-GB/firefox/addon/simplelogin/">
+<a href="https://addons.mozilla.org/firefox/addon/simplelogin/">
 <img src="https://img.shields.io/amo/rating/simplelogin?label=Firefox%20Add-On&logo=SimpleLogin">
 </a>
 
-<a href="https://stats.uptimerobot.com/APkzziNWoM">
-<img src="https://img.shields.io/uptimerobot/ratio/7/m782965045-15d8e413b20b5376f58db050">
-</a>
-
 <a href="./LICENSE">
 <img src="https://img.shields.io/github/license/simple-login/app">
 </a>
 
-<a href="https://twitter.com/simple_login">
-<img src="https://img.shields.io/twitter/follow/simple_login?style=social">
+<a href="https://twitter.com/simplelogin">
+<img src="https://img.shields.io/twitter/follow/simplelogin?style=social">
 </a>
 
 </p>
 
-> Yet another email forwarding service?
-
-In some way yes... However, SimpleLogin is a bit different because:
-
-- Fully open source: both the server and client code (browser extension, JS library) are open source so anyone can freely inspect and (hopefully) improve the code.
-
-- The only email forwarding solution that is **self-hostable**: with our detailed self-hosting instructions and most of components running as Docker container, anyone who knows `ssh` is able to deploy SimpleLogin on their server.
-
-- Not just email alias: SimpleLogin is a privacy-first and developer-friendly identity provider that:
-    - offers privacy for users
-    - is simple to use for developers. SimpleLogin is a privacy-focused alternative to the "Login with Facebook/Google/Twitter" buttons.
-
-- Plenty of features: browser extension, custom domain, catch-all alias, OAuth libraries, etc.
-
-- Open roadmap at https://trello.com/b/4d6A69I4/open-roadmap: you know the exciting features we are working on.
-
-At the heart of SimpleLogin is `email alias`: an alias is a normal email address but all emails sent to an alias are **forwarded** to your email inbox. SimpleLogin alias can also **send** emails: for your contact, the alias is therefore your email address. Use alias whenever you need to give out your email address to protect your online identity. More info on our website at https://simplelogin.io
-
 <p align="center">
-    <img src="./docs/custom-alias.png" height="150px">
+    <a href="https://simplelogin.io">
+        <img src="./docs/hero.png" height="600px">
+    </a>
 </p>
 
-# Quick start
+---
 
-If you have Docker installed, run the following command to start SimpleLogin local server:
+Your email address is your **online identity**. When you use the same email address everywhere, you can be easily tracked.
+More information on https://simplelogin.io
 
+This README contains instructions on how to self host SimpleLogin.
 
-```bash
-docker run --name sl -it --rm \
-    -e RESET_DB=true \
-    -e CONFIG=/code/example.env \
-    -p 7777:7777 \
-    simplelogin/app:2.0.0 python server.py
-```
+Once you have your own SimpleLogin instance running, you can change the `API URL` in SimpleLogin's Chrome/Firefox extension, Android/iOS app to your server.
 
-Then open http://localhost:7777, you should be able to login with `john@wick.com/password` account!
-
-To use SimpleLogin aliases, you need to deploy it on your server with some DNS setup though,
-the following section will show a step-by-step guide on how to get your own email forwarder service!
-
-# Table of Contents
-
-[1. General Architecture](#general-architecture)
-
-[2. Self Hosting](#self-hosting)
-
-[3. Contributing Guide](#contributing)
-
-
-## General Architecture
-
-<p align="center">
-    <img src="./docs/archi.png" height="450px">
-</p>
-
-SimpleLogin backend consists of 2 main components:
-
-- the `webapp` used by several clients: web UI (the dashboard), browser extension (Chrome & Firefox for now), OAuth clients (apps that integrate "Login with SimpleLogin" button) and mobile app (work in progress).
-
-- the `email handler`: implements the email forwarding (i.e. alias receiving email) and email sending (i.e. alias sending email).
-
-## Self hosting
+SimpleLogin roadmap is at https://github.com/simple-login/app/projects/1 and our forum at https://github.com/simple-login/app/discussions, feel free to submit new ideas or vote on features.
 
 ### Prerequisites
 
@@ -98,9 +44,6 @@ SimpleLogin backend consists of 2 main components:
 
 - a domain that you can config the DNS. It could be a sub-domain. In the rest of the doc, let's say it's `mydomain.com` for the email and `app.mydomain.com` for SimpleLogin webapp. Please make sure to replace these values by your domain name whenever they appear in the doc. A trick we use is to download this README file on your computer and replace all `mydomain.com` occurrences by your domain.
 
-- [Optional] AWS S3, Sentry, Google/Facebook/Github developer accounts. These are necessary only if you want to activate these options.
-
-
 Except for the DNS setup that is usually done on your domain registrar interface, all the below steps are to be done on your server. The commands are to run with `bash` (or any bash-compatible shell like `zsh`) being the shell. If you use other shells like `fish`, please make sure to adapt the commands.
 
 ### Some utility packages
@@ -108,9 +51,17 @@ Except for the DNS setup that is usually done on your domain registrar interface
 These packages are used to verify the setup. Install them by:
 
 ```bash
-sudo apt install -y dnsutils
+sudo apt update && sudo apt install -y dnsutils
 ```
 
+Create a directory to store SimpleLogin data:
+
+```bash
+mkdir sl
+mkdir sl/pgp # to store PGP key
+mkdir sl/db # to store database
+mkdir sl/upload # to store quarantine emails
+```
 
 ### DKIM
 
@@ -123,7 +74,7 @@ Setting up DKIM is highly recommended to reduce the chance your emails ending up
 First you need to generate a private and public key for DKIM:
 
 ```bash
-openssl genrsa -out dkim.key 1024
+openssl genrsa -out dkim.key -traditional 1024
 openssl rsa -in dkim.key -pubout -out dkim.pub.key
 ```
 
@@ -152,7 +103,9 @@ mydomain.com.	3600	IN	MX	10 app.mydomain.com.
 ```
 
 #### A record
-An **A record** that points `app.mydomain.com.` to your server IP. To verify, the following command
+An **A record** that points `app.mydomain.com.` to your server IP.
+If you are using CloudFlare, we recommend to disable the "Proxy" option.
+To verify, the following command
 
 ```bash
 dig @1.1.1.1 app.mydomain.com a
@@ -187,7 +140,7 @@ then the `PUBLIC_KEY` would be `abcdefgh`.
 You can get the `PUBLIC_KEY` by running this command:
 
 ```bash
-sed "s/-----BEGIN PUBLIC KEY-----/v=DKIM1; k=rsa; p=/g" dkim.pub.key | sed 's/-----END PUBLIC KEY-----//g' |tr -d '\n'
+sed "s/-----BEGIN PUBLIC KEY-----/v=DKIM1; k=rsa; p=/g" $(pwd)/dkim.pub.key | sed 's/-----END PUBLIC KEY-----//g' |tr -d '\n' | awk 1
 ```
 
 To verify, the following command
@@ -208,7 +161,7 @@ Similar to DKIM, setting up SPF is highly recommended.
 Add a TXT record for `mydomain.com.` with the value:
 
 ```
-v=spf1 mx -all
+v=spf1 mx ~all
 ```
 
 What it means is only your server can send email with `@mydomain.com` domain.
@@ -251,11 +204,10 @@ Now the boring DNS stuffs are done, let's do something more fun!
 
 If you don't already have Docker installed on your server, please follow the steps on [Docker CE for Ubuntu](https://docs.docker.com/v17.12/install/linux/docker-ce/ubuntu/) to install Docker.
 
-Tips: if you are not using `root` user and you want to run Docker without the `sudo` prefix, add your account to `docker` group with the following command.
-You might need to exit and ssh again to your server for this to be taken into account.
+You can also install Docker using the [docker-install](https://github.com/docker/docker-install) script which is
 
 ```bash
-sudo usermod -a -G docker $USER
+curl -fsSL https://get.docker.com | sh
 ```
 
 ### Prepare the Docker network
@@ -265,8 +217,8 @@ Later, we will setup Postfix to authorize this network.
 
 ```bash
 sudo docker network create -d bridge \
-    --subnet=240.0.0.0/24 \
-    --gateway=240.0.0.1 \
+    --subnet=10.0.0.0/24 \
+    --gateway=10.0.0.1 \
     sl-network
 ```
 
@@ -279,12 +231,13 @@ If you already have a Postgres database in use, you can skip this section and ju
 Run a Postgres Docker container as your Postgres database server. Make sure to replace `myuser` and `mypassword` with something more secret.
 
 ```bash
-sudo docker run -d \
+docker run -d \
     --name sl-db \
     -e POSTGRES_PASSWORD=mypassword \
     -e POSTGRES_USER=myuser \
     -e POSTGRES_DB=simplelogin \
-    -p 5432:5432 \
+    -p 127.0.0.1:5432:5432 \
+    -v $(pwd)/sl/db:/var/lib/postgresql/data \
     --restart always \
     --network="sl-network" \
     postgres:12.1
@@ -293,7 +246,7 @@ sudo docker run -d \
 To test whether the database operates correctly or not, run the following command:
 
 ```bash
-sudo docker exec -it sl-db psql -U myuser simplelogin
+docker exec -it sl-db psql -U myuser simplelogin
 ```
 
 you should be logged in the postgres console. Type `exit` to exit postgres console.
@@ -308,6 +261,9 @@ sudo apt-get install -y postfix postfix-pgsql -y
 
 Choose "Internet Site" in Postfix installation window then keep using the proposed value as *System mail name* in the next window.
 
+![](./docs/postfix-installation.png)
+![](./docs/postfix-installation2.png)
+
 Replace `/etc/postfix/main.cf` with the following content. Make sure to replace `mydomain.com` by your domain.
 
 ```
@@ -330,17 +286,19 @@ compatibility_level = 2
 # TLS parameters
 smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
 smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
-smtpd_use_tls=yes
 smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+smtp_tls_security_level = may
+smtpd_tls_security_level = may
 
 # See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for
 # information on enabling SSL in the smtp client.
 
 alias_maps = hash:/etc/aliases
-mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 240.0.0.0/24
+mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 10.0.0.0/24
 
 # Set your domain here
+mydestination =
 myhostname = app.mydomain.com
 mydomain = mydomain.com
 myorigin = mydomain.com
@@ -376,8 +334,14 @@ smtpd_recipient_restrictions =
    permit
 ```
 
+Check that the ssl certificates `/etc/ssl/certs/ssl-cert-snakeoil.pem` and `/etc/ssl/private/ssl-cert-snakeoil.key` exist. Depending on the linux distribution you are using they may or may not be present. If they are not, you will need to generate them with this command:
+
+```bash
+openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/private/ssl-cert-snakeoil.key -out /etc/ssl/certs/ssl-cert-snakeoil.pem
+```
+
 Create the `/etc/postfix/pgsql-relay-domains.cf` file with the following content.
-Make sure that the database config is correctly set and replace `mydomain.com` with your domain.
+Make sure that the database config is correctly set, replace `mydomain.com` with your domain, update 'myuser' and 'mypassword' with your postgres credentials.
 
 ```
 # postgres config
@@ -391,7 +355,7 @@ query = SELECT domain FROM custom_domain WHERE domain='%s' AND verified=true
 ```
 
 Create the `/etc/postfix/pgsql-transport-maps.cf` file with the following content.
-Again, make sure that the database config is correctly set and replace `mydomain.com` with your domain.
+Again, make sure that the database config is correctly set, replace `mydomain.com` with your domain, update 'myuser' and 'mypassword' with your postgres credentials.
 
 ```
 # postgres config
@@ -411,14 +375,16 @@ Finally, restart Postfix
 sudo systemctl restart postfix
 ```
 
-
 ### Run SimpleLogin Docker containers
 
-To run the server, you need a config file. Please have a look at [config example](example.env) for an example to create one. Some parameters are optional and are commented out by default. Some have "dummy" values, fill them up if you want to enable these features (Paddle, AWS, etc).
+To run SimpleLogin, you need a config file at `$(pwd)/simplelogin.env`. Below is an example that you can use right away, make sure to
 
-Let's put your config file at `~/simplelogin.env`. Below is an example that you can use right away, make sure to replace `mydomain.com` by your domain and set `FLASK_SECRET` to a secret string.
+- replace `mydomain.com` by your domain,
+- set `FLASK_SECRET` to a secret string,
+- update 'myuser' and 'mypassword' with your database credentials used in previous step.
 
-Make sure to update the following variables and replace these values by yours.
+All possible parameters can be found in [config example](example.env). Some are optional and are commented out by default.
+Some have "dummy" values, fill them up if you want to enable these features (Paddle, AWS, etc).
 
 ```.env
 # WebApp URL
@@ -437,68 +403,101 @@ EMAIL_SERVERS_WITH_PRIORITY=[(10, "app.mydomain.com.")]
 # this option doesn't make sense in self-hosted. Set this variable to disable this option.
 DISABLE_ALIAS_SUFFIX=1
 
-# If you want to use another MTA to send email, you could set the address of your MTA here
-# By default, emails are sent using the the same Postfix server that receives emails
-# POSTFIX_SERVER=my-postfix.com
-
 # the DKIM private key used to compute DKIM-Signature
 DKIM_PRIVATE_KEY_PATH=/dkim.key
 
-# the DKIM public key used to setup custom domain DKIM
-DKIM_PUBLIC_KEY_PATH=/dkim.pub.key
-
 # DB Connection
 DB_URI=postgresql://myuser:mypassword@sl-db:5432/simplelogin
 
 FLASK_SECRET=put_something_secret_here
+
+GNUPGHOME=/sl/pgp
+
+LOCAL_FILE_UPLOAD=1
+
+POSTFIX_SERVER=10.0.0.1
 ```
 
 
 Before running the webapp, you need to prepare the database by running the migration:
 
 ```bash
-sudo docker run --rm \
+docker run --rm \
     --name sl-migration \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
     -v $(pwd)/dkim.key:/dkim.key \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
     -v $(pwd)/simplelogin.env:/code/.env \
     --network="sl-network" \
-    simplelogin/app:2.0.0 flask db upgrade
+    simplelogin/app:3.4.0 flask db upgrade
 ```
 
 This command could take a while to download the `simplelogin/app` docker image.
 
-Now, it's time to run the `webapp` container!
+Init data
 
 ```bash
-sudo docker run -d \
-    --name sl-app \
+docker run --rm \
+    --name sl-init \
+    -v $(pwd)/sl:/sl \
     -v $(pwd)/simplelogin.env:/code/.env \
     -v $(pwd)/dkim.key:/dkim.key \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
-    -p 7777:7777 \
+    --network="sl-network" \
+    simplelogin/app:3.4.0 python init_app.py
+```
+
+Now, it's time to run the `webapp` container!
+
+```bash
+docker run -d \
+    --name sl-app \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
+    -v $(pwd)/simplelogin.env:/code/.env \
+    -v $(pwd)/dkim.key:/dkim.key \
+    -v $(pwd)/dkim.pub.key:/dkim.pub.key \
+    -p 127.0.0.1:7777:7777 \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:2.0.0
+    simplelogin/app:3.4.0
 ```
 
 Next run the `email handler`
 
 ```bash
-sudo docker run -d \
+docker run -d \
     --name sl-email \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
     -v $(pwd)/simplelogin.env:/code/.env \
     -v $(pwd)/dkim.key:/dkim.key \
     -v $(pwd)/dkim.pub.key:/dkim.pub.key \
-    -p 20381:20381 \
+    -p 127.0.0.1:20381:20381 \
     --restart always \
     --network="sl-network" \
-    simplelogin/app:2.0.0 python email_handler.py
+    simplelogin/app:3.4.0 python email_handler.py
+```
+
+And finally the `job runner`
+
+```bash
+docker run -d \
+    --name sl-job-runner \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
+    -v $(pwd)/simplelogin.env:/code/.env \
+    -v $(pwd)/dkim.key:/dkim.key \
+    -v $(pwd)/dkim.pub.key:/dkim.pub.key \
+    --restart always \
+    --network="sl-network" \
+    simplelogin/app:3.4.0 python job_runner.py
 ```
 
 ### Nginx
 
-Install Nginx
+Install Nginx and make sure to replace `mydomain.com` by your domain
 
 ```bash
 sudo apt-get install -y nginx
@@ -506,510 +505,66 @@ sudo apt-get install -y nginx
 
 Then, create `/etc/nginx/sites-enabled/simplelogin` with the following lines:
 
-```
+```nginx
 server {
     server_name  app.mydomain.com;
 
     location / {
-        proxy_pass http://localhost:7777;
+        proxy_pass              http://localhost:7777;
+    	proxy_set_header        Host $host;
     }
 }
 ```
 
+Note: If `/etc/nginx/sites-enabled/default` exists, delete it or certbot will fail due to the conflict. The `simplelogin` file should be the only file in `sites-enabled`.
+
 Reload Nginx with the command below
 
 ```bash
 sudo systemctl reload nginx
 ```
 
-At this step, you should also setup the SSL for Nginx. [Certbot](https://certbot.eff.org/lets-encrypt/ubuntuxenial-nginx) can be a good option if you want a free SSL certificate.
+At this step, you should also setup the SSL for Nginx. [Here's our guide how](./docs/ssl.md).
 
 ### Enjoy!
 
-If all of the above steps are successful, open http://app.mydomain.com/ and create your first account!
+If all the above steps are successful, open http://app.mydomain.com/ and create your first account!
 
 By default, new accounts are not premium so don't have unlimited alias. To make your account premium,
-please go to the database, table "users" and set "lifetime" column to "1" or "TRUE".
+please go to the database, table "users" and set "lifetime" column to "1" or "TRUE":
+
+```
+docker exec -it sl-db psql -U myuser simplelogin
+UPDATE users SET lifetime = TRUE;
+exit
+```
+
+Once you've created all your desired login accounts, add these lines to `/simplelogin.env` to disable further registrations:
+
+```
+DISABLE_REGISTRATION=1
+DISABLE_ONBOARDING=true
+```
+
+Then restart the web app to apply: `docker restart sl-app`
+
+### Donations Welcome
 
 You don't have to pay anything to SimpleLogin to use all its features.
-You could make a donation to SimpleLogin on our Patreon page at https://www.patreon.com/simplelogin if you wish though.
+If you like the project, you can make a donation on our Open Collective page at https://opencollective.com/simplelogin
 
 ### Misc
 
 The above self-hosting instructions correspond to a freshly Ubuntu server and doesn't cover all possible server configuration.
 Below are pointers to different topics:
 
+- [Troubleshooting](docs/troubleshooting.md)
+- [Enable SSL](docs/ssl.md)
 - [UFW - uncomplicated firewall](docs/ufw.md)
 - [SES - Amazon Simple Email Service](docs/ses.md)
 - [Upgrade existing SimpleLogin installation](docs/upgrade.md)
-
-## Contributing
-
-All work on SimpleLogin happens directly on GitHub.
-
-### Run code locally
-
-The project uses Python 3.7+ and Node v10. First, install all dependencies by running the following command. Feel free to use `virtualenv` or similar tools to isolate development environment.
-
-```bash
-pip3 install -r requirements.txt
-```
-
-You also need to install `gpg`, on Mac it can be done with:
-
-```bash
-brew install gnupg
-```
-
-
-Then make sure all tests pass
-
-```bash
-pytest
-```
-
-Install npm packages
-
-```bash
-cd static && npm install
-```
-
-To run the code locally, please create a local setting file based on `example.env`:
-
-```
-cp example.env .env
-```
-
-Make sure to uncomment the `RESET_DB=true` to create the database locally.
-
-Feel free to custom your `.env` file, it would be your default setting when developing locally. This file is ignored by git.
-
-You don't need all the parameters, for example, if you don't update images to s3, then
-`BUCKET`, `AWS_ACCESS_KEY_ID` can be empty or if you don't use login with Github locally, `GITHUB_CLIENT_ID` doesn't have to be filled. The `example.env` file contains minimal requirement so that if you run:
-
-```
-python3 server.py
-```
-
-then open http://localhost:7777, you should be able to login with the following account
-
-```
-john@wick.com / password
-```
-
-### API
-
-SimpleLogin current API clients are Chrome/Firefox/Safari extension and mobile (iOS/Android) app.
-These clients rely on `API Code` for authentication.
-
-Once the `Api Code` is obtained, either via user entering it (in Browser extension case) or by logging in (in Mobile case),
-the client includes the `api code` in `Authentication` header in almost all requests.
-
-For some endpoints, the `hostname` should be passed in query string. `hostname` is the the URL hostname (cf https://en.wikipedia.org/wiki/URL), for ex if URL is http://www.example.com/index.html then the hostname is `www.example.com`. This information is important to know where an alias is used in order to suggest user the same alias if they want to create on alias on the same website in the future.
-
-If error, the API returns 4** with body containing the error message, for example:
-
-```json
-{
-  "error":  "request body cannot be empty"
-}
-```
-
-The error message could be displayed to user as-is, for example for when user exceeds their alias quota.
-Some errors should be fixed during development however: for example error like `request body cannot be empty` is there to catch development error and should never be shown to user.
-
-All following endpoint return `401` status code if the API Key is incorrect.
-
-#### GET /api/user_info
-
-Given the API Key, return user name and whether user is premium.
-This endpoint could be used to validate the api key.
-
-Input:
-- `Authentication` header that contains the api key
-
-Output: if api key is correct, return a json with user name and whether user is premium, for example:
-
-```json
-{
-	"name": "John Wick",
-	"is_premium": false
-}
-```
-
-If api key is incorrect, return 401.
-
-
-#### GET /api/v2/alias/options
-
-User alias info and suggestion. Used by the first extension screen when user opens the extension.
-
-Input:
-- `Authentication` header that contains the api key
-- (Optional but recommended) `hostname` passed in query string.
-
-Output: a json with the following field:
-- can_create: boolean. Whether user can create new alias
-- suffixes: list of string. List of alias `suffix` that user can use. If user doesn't have custom domain, this list has a single element which is the alias default domain (simplelogin.co).
-- prefix_suggestion: string. Suggestion for the `alias prefix`. Usually this is the website name extracted from `hostname`. If no `hostname`, then the `prefix_suggestion` is empty.
-- existing: list of string. List of existing alias.
-- recommendation: optional field, dictionary. If an alias is already used for this website, the recommendation will be returned. There are 2 subfields in `recommendation`: `alias` which is the recommended alias and `hostname` is the website on which this alias is used before.
-
-For ex:
-```json
-{
-    "can_create": true,
-    "existing": [
-        "my-first-alias.meo@sl.local",
-        "e1.cat@sl.local",
-        "e2.chat@sl.local",
-        "e3.cat@sl.local"
-    ],
-    "prefix_suggestion": "test",
-    "recommendation": {
-        "alias": "e1.cat@sl.local",
-        "hostname": "www.test.com"
-    },
-    "suffixes": [
-        "@very-long-domain.com.net.org",
-        "@ab.cd",
-        ".cat@sl.local"
-    ]
-}
-```
-
-#### POST /api/alias/custom/new
-
-Create a new custom alias.
-
-Input:
-- `Authentication` header that contains the api key
-- (Optional but recommended) `hostname` passed in query string
-- Request Message Body in json (`Content-Type` is `application/json`)
-    - alias_prefix: string. The first part of the alias that user can choose.
-    - alias_suffix: should be one of the suffixes returned in the `GET /api/v2/alias/options` endpoint.
-    - (Optional) note: alias note
-
-Output:
-If success, 201 with the new alias, for example
-
-```json
-{
-  "alias": "www_groupon_com@my_domain.com"
-}
-```
-
-#### POST /api/alias/random/new
-
-Create a new random alias.
-
-Input:
-- `Authentication` header that contains the api key
-- (Optional but recommended) `hostname` passed in query string
-- (Optional) mode: either `uuid` or `word`. By default, use the user setting when creating new random alias.
-- Request Message Body in json (`Content-Type` is `application/json`)
-    - (Optional) note: alias note
-
-Output:
-If success, 201 with the new alias, for example
-
-```json
-{
-  "alias": "www_groupon_com@my_domain.com"
-}
-```
-
-#### POST /api/auth/login
-
-Input:
-- email
-- password
-- device: device name. Used to create the API Key. Should be humanly readable so user can manage later on the "API Key" page.
-
-Output:
-- name: user name, could be an empty string
-- mfa_enabled: boolean
-- mfa_key: only useful when user enables MFA. In this case, user needs to enter their OTP token in order to login.
-- api_key: if MFA is not enabled, the `api key` is returned right away.
-
-The `api_key` is used in all subsequent requests. It's empty if MFA is enabled.
-If user hasn't enabled MFA, `mfa_key` is empty.
-
-#### POST /api/auth/mfa
-
-Input:
-- mfa_token: OTP token that user enters
-- mfa_key: MFA key obtained in previous auth request, e.g. /api/auth/login
-- device: the device name, used to create an ApiKey associated with this device
-
-Output:
-- name: user name, could be an empty string
-- api_key: if MFA is not enabled, the `api key` is returned right away.
-
-The `api_key` is used in all subsequent requests. It's empty if MFA is enabled.
-If user hasn't enabled MFA, `mfa_key` is empty.
-
-#### POST /api/auth/facebook
-
-Input:
-- facebook_token: Facebook access token
-- device: device name. Used to create the API Key. Should be humanly readable so user can manage later on the "API Key" page.
-
-Output: Same output as for `/api/auth/login` endpoint
-
-
-#### POST /api/auth/google
-
-Input:
-- google_token: Google access token
-- device: device name. Used to create the API Key. Should be humanly readable so user can manage later on the "API Key" page.
-
-Output: Same output as for `/api/auth/login` endpoint
-
-
-#### POST /api/auth/register
-
-Input:
-- email
-- password
-
-Output: 200 means user is going to receive an email that contains an *activation code*. User needs to enter this code to confirm their account -> next endpoint.
-
-
-#### POST /api/auth/activate
-
-Input:
-- email
-- code: the activation code
-
-Output:
-- 200: account is activated. User can login now
-- 400: wrong email, code
-- 410: wrong code too many times. User needs to ask for an reactivation -> next endpoint
-
-#### POST /api/auth/reactivate
-
-Input:
-- email
-
-Output:
-- 200: user is going to receive an email that contains the activation code.
-
-#### GET /api/aliases
-
-Get user aliases.
-
-Input:
-- `Authentication` header that contains the api key
-- `page_id` used for the pagination. The endpoint returns maximum 20 aliases for each page. `page_id` starts at 0.
-
-Output:
-If success, 200 with the list of aliases, for example:
-
-```json
-{
-    "aliases": [
-        {
-            "creation_date": "2020-02-04 16:23:02+00:00",
-            "creation_timestamp": 1580833382,
-            "email": "e3@.alo@sl.local",
-            "id": 4,
-            "nb_block": 0,
-            "nb_forward": 0,
-            "nb_reply": 0,
-            "enabled": true,
-            "note": "This is a note"
-        },
-        {
-            "creation_date": "2020-02-04 16:23:02+00:00",
-            "creation_timestamp": 1580833382,
-            "email": "e2@.meo@sl.local",
-            "id": 3,
-            "nb_block": 0,
-            "nb_forward": 0,
-            "nb_reply": 0,
-            "enabled": false,
-            "note": null
-        }
-    ]
-}
-```
-
-#### DELETE /api/aliases/:alias_id
-
-Delete an alias
-
-Input:
-- `Authentication` header that contains the api key
-- `alias_id` in url.
-
-Output:
-If success, 200.
-
-
-```json
-{
-    "deleted": true
-}
-```
-
-#### POST /api/aliases/:alias_id/toggle
-
-Enable/disable alias
-
-Input:
-- `Authentication` header that contains the api key
-- `alias_id` in url.
-
-Output:
-If success, 200 along with the new alias status:
-
-```json
-{
-    "enabled": false
-}
-```
-
-#### GET /api/aliases/:alias_id/activities
-
-Get activities for a given alias.
-
-Input:
-- `Authentication` header that contains the api key
-- `alias_id`: the alias id, passed in url.
-- `page_id` used in request query (`?page_id=0`). The endpoint returns maximum 20 aliases for each page. `page_id` starts at 0.
-
-Output:
-If success, 200 with the list of activities, for example:
-
-```json
-{
-  "activities": [
-    {
-      "action": "reply",
-      "from": "yes_meo_chat@sl.local",
-      "timestamp": 1580903760,
-      "to": "marketing@example.com"
-    },
-    {
-      "action": "reply",
-      "from": "yes_meo_chat@sl.local",
-      "timestamp": 1580903760,
-      "to": "marketing@example.com"
-    }
-  ]
-}
-```
-
-
-### Database migration
-
-The database migration is handled by `alembic`
-
-Whenever the model changes, a new migration has to be created
-
-Set the database connection to use a current database (i.e. the one without the model changes you just made), for example, if you have a staging config at `~/config/simplelogin/staging.env`, you can do:
-
-```bash
-ln -sf ~/config/simplelogin/staging.env .env
-```
-
-Generate the migration script and make sure to review it before committing it. Sometimes (very rarely though), the migration generation can go wrong.
-
-```bash
-flask db migrate
-```
-
-In local the database creation in Sqlite doesn't use migration and uses directly `db.create_all()` (cf `fake_data()` method). This is because Sqlite doesn't handle well the migration. As sqlite is only used during development, the database is deleted and re-populated at each run.
-
-### Code structure
-
-The repo consists of the three following entry points:
-
-- wsgi.py and server.py: the webapp.
-- email_handler.py: the email handler.
-- cron.py: the cronjob.
-
-Here are the small sum-ups of the directory structures and their roles:
-
-- app/: main Flask app. It is structured into different packages representing different features like oauth,  api, dashboard, etc.
-- local_data/: contains files to facilitate the local development. They are replaced during the deployment.
-- migrations/: generated by flask-migrate. Edit these files will be only edited when you spot (very rare) errors on the database migration files.
-- static/: files available at `/static` url.
-- templates/: contains both html and email templates.
-- tests/: tests. We don't really distinguish unit, functional or integration test. A test is simply here to make sure a feature works correctly.
-
-The code is formatted using https://github.com/psf/black, to format the code, simply run
-
-```
-black .
-```
-
-### OAuth flow
-
-SL currently supports code and implicit flow.
-
-#### Code flow
-
-To trigger the code flow locally, you can go to the following url after running `python server.py`:
-
-```
-http://localhost:7777/oauth/authorize?client_id=client-id&state=123456&response_type=code&redirect_uri=http%3A%2F%2Flocalhost%3A7000%2Fcallback&state=random_string
-```
-
-You should see there the authorization page where user is asked for permission to share their data. Once user approves, user is redirected to this url with an `authorization code`: `http://localhost:7000/callback?state=123456&code=the_code`
-
-Next, exchange the code to get the token with `{code}` replaced by the code obtained in previous step. The `http` tool used here is https://httpie.org
-
-```
-http -f -a client-id:client-secret http://localhost:7777/oauth/token grant_type=authorization_code code={code}
-```
-
-This should return an `access token` that allows to get user info via the following command. Again, `http` tool is used.
-
-```
-http http://localhost:7777/oauth/user_info 'Authorization:Bearer {token}'
-```
-
-#### Implicit flow
-
-Similar to code flow, except for the the `access token` which we we get back with the redirection.
-For implicit flow, the url is
-
-```
-http://localhost:7777/oauth/authorize?client_id=client-id&state=123456&response_type=token&redirect_uri=http%3A%2F%2Flocalhost%3A7000%2Fcallback&state=random_string
-```
-
-#### OpenID and OAuth2 response_type & scope
-
-According to the sharing web blog titled [Diagrams of All The OpenID Connect Flows](https://medium.com/@darutk/diagrams-of-all-the-openid-connect-flows-6968e3990660), we should pay attention to:
-
-- `response_type` can be either `code, token, id_token`  or any combination of those attributes.
-- `scope` might contain `openid`
-
-Below are the potential combinations that are taken into account in SL until now:
-
-```
-response_type=code
-    scope:
-	    with `openid` in scope, return `id_token` at /token: OK
-	    without: OK
-
-response_type=token
-    scope:
-	    with and without `openid`, nothing to do: OK
-
-response_type=id_token
-    return `id_token` in /authorization endpoint
-
-response_type=id_token token
-    return `id_token` in addition to `access_token` in /authorization endpoint
-
-response_type=id_token code
-    return `id_token` in addition to `authorization_code` in /authorization endpoint
-
-```
-
+- [Enforce SPF](docs/enforce-spf.md)
+- [Postfix TLS](docs/postfix-tls.md)
 
 ## ❤️ Contributors
 
@@ -1022,5 +577,8 @@ Thanks go to these wonderful people:
     <td align="center"><a href="https://github.com/NinhDinh"><img src="https://avatars2.githubusercontent.com/u/1419742?s=460&v=4" width="100px;" alt="Ninh Dinh"/><br /><sub><b>Ninh Dinh</b></sub></a><br /></td>
     <td align="center"><a href="https://github.com/ntung"><img src="https://avatars1.githubusercontent.com/u/663341?s=460&v=4" width="100px;" alt="Tung Nguyen V. N."/><br /><sub><b>Tung Nguyen V. N.</b></sub></a><br /></td>
     <td align="center"><a href="https://www.linkedin.com/in/nguyenkims/"><img src="https://simplelogin.io/about/me.jpeg" width="100px;" alt="Son Nguyen Kim"/><br /><sub><b>Son Nguyen Kim</b></sub></a><br /></td>
+    <td align="center"><a href="https://github.com/developStorm"><img src="https://avatars1.githubusercontent.com/u/59678453?s=460&u=3813d29a125b3edeb44019234672b704f7b9b76a&v=4" width="100px;" alt="Raymond Nook"/><br /><sub><b>Raymond Nook</b></sub></a><br /></td>
+    <td align="center"><a href="https://github.com/SibrenVasse"><img src="https://avatars1.githubusercontent.com/u/5833571?s=460&u=78aea62ffc215885a0319437fc629a7596ddea31&v=4" width="100px;" alt="Sibren Vasse"/><br /><sub><b>Sibren Vasse</b></sub></a><br /></td>
+    <td align="center"><a href="https://github.com/TheLastProject"><img src="https://avatars.githubusercontent.com/u/1885159?s=460&u=ebeeb346c4083c0d493a134f4774f925d3437f98&v=4" width="100px;" alt="Sylvia van Os"/><br /><sub><b>Sylvia van Os</b></sub></a><br /></td>
 </tr>
 </table>

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+We only add security updates to the latest MAJOR.MINOR version of the project. No security updates are backported to previous versions.
+If you want be up to date on security patches, make sure your SimpleLogin image is up to date.
+
+## Reporting a Vulnerability
+
+If you've found a security vulnerability, you can disclose it responsibly by sending a summary to security@simplelogin.io.
+We will review the potential threat and fix it as fast as we can.
+
+We are incredibly thankful for people who disclose vulnerabilities, unfortunately we do not have a bounty program in place yet.
+

alembic.ini

@@ -0,0 +1,83 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration files
+file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks=black
+# black.type=console_scripts
+# black.entrypoint=black
+# black.options=-l 79
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

app/account_linking.py

@@ -0,0 +1,316 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+from arrow import Arrow
+from newrelic import agent
+from sqlalchemy import or_
+
+from app.db import Session
+from app.email_utils import send_welcome_email
+from app.utils import sanitize_email, canonicalize_email
+from app.errors import (
+    AccountAlreadyLinkedToAnotherPartnerException,
+    AccountIsUsingAliasAsEmail,
+    AccountAlreadyLinkedToAnotherUserException,
+)
+from app.log import LOG
+from app.models import (
+    PartnerSubscription,
+    Partner,
+    PartnerUser,
+    User,
+    Alias,
+)
+from app.utils import random_string
+
+
+class SLPlanType(Enum):
+    Free = 1
+    Premium = 2
+
+
+@dataclass
+class SLPlan:
+    type: SLPlanType
+    expiration: Optional[Arrow]
+
+
+@dataclass
+class PartnerLinkRequest:
+    name: str
+    email: str
+    external_user_id: str
+    plan: SLPlan
+    from_partner: bool
+
+
+@dataclass
+class LinkResult:
+    user: User
+    strategy: str
+
+
+def set_plan_for_partner_user(partner_user: PartnerUser, plan: SLPlan):
+    sub = PartnerSubscription.get_by(partner_user_id=partner_user.id)
+    if plan.type == SLPlanType.Free:
+        if sub is not None:
+            LOG.i(
+                f"Deleting partner_subscription [user_id={partner_user.user_id}] [partner_id={partner_user.partner_id}]"
+            )
+            PartnerSubscription.delete(sub.id)
+            agent.record_custom_event("PlanChange", {"plan": "free"})
+    else:
+        if sub is None:
+            LOG.i(
+                f"Creating partner_subscription [user_id={partner_user.user_id}] [partner_id={partner_user.partner_id}]"
+            )
+            PartnerSubscription.create(
+                partner_user_id=partner_user.id,
+                end_at=plan.expiration,
+            )
+            agent.record_custom_event("PlanChange", {"plan": "premium", "type": "new"})
+        else:
+            if sub.end_at != plan.expiration:
+                LOG.i(
+                    f"Updating partner_subscription [user_id={partner_user.user_id}] [partner_id={partner_user.partner_id}]"
+                )
+                agent.record_custom_event(
+                    "PlanChange", {"plan": "premium", "type": "extension"}
+                )
+                sub.end_at = plan.expiration
+    Session.commit()
+
+
+def set_plan_for_user(user: User, plan: SLPlan, partner: Partner):
+    partner_user = PartnerUser.get_by(partner_id=partner.id, user_id=user.id)
+    if partner_user is None:
+        return
+    return set_plan_for_partner_user(partner_user, plan)
+
+
+def ensure_partner_user_exists_for_user(
+    link_request: PartnerLinkRequest, sl_user: User, partner: Partner
+) -> PartnerUser:
+    # Find partner_user by user_id
+    res = PartnerUser.get_by(user_id=sl_user.id)
+    if res and res.partner_id != partner.id:
+        raise AccountAlreadyLinkedToAnotherPartnerException()
+    if not res:
+        res = PartnerUser.create(
+            user_id=sl_user.id,
+            partner_id=partner.id,
+            partner_email=link_request.email,
+            external_user_id=link_request.external_user_id,
+        )
+        Session.commit()
+        LOG.i(
+            f"Created new partner_user for partner:{partner.id} user:{sl_user.id} external_user_id:{link_request.external_user_id}. PartnerUser.id is {res.id}"
+        )
+    return res
+
+
+class ClientMergeStrategy(ABC):
+    def __init__(
+        self,
+        link_request: PartnerLinkRequest,
+        user: Optional[User],
+        partner: Partner,
+    ):
+        if self.__class__ == ClientMergeStrategy:
+            raise RuntimeError("Cannot directly instantiate a ClientMergeStrategy")
+        self.link_request = link_request
+        self.user = user
+        self.partner = partner
+
+    @abstractmethod
+    def process(self) -> LinkResult:
+        pass
+
+
+class NewUserStrategy(ClientMergeStrategy):
+    def process(self) -> LinkResult:
+        # Will create a new SL User with a random password
+        canonical_email = canonicalize_email(self.link_request.email)
+        new_user = User.create(
+            email=canonical_email,
+            name=self.link_request.name,
+            password=random_string(20),
+            activated=True,
+            from_partner=self.link_request.from_partner,
+        )
+        partner_user = PartnerUser.create(
+            user_id=new_user.id,
+            partner_id=self.partner.id,
+            external_user_id=self.link_request.external_user_id,
+            partner_email=self.link_request.email,
+        )
+        LOG.i(
+            f"Created new user for login request for partner:{self.partner.id} external_user_id:{self.link_request.external_user_id}. New user {new_user.id} partner_user:{partner_user.id}"
+        )
+        set_plan_for_partner_user(
+            partner_user,
+            self.link_request.plan,
+        )
+        Session.commit()
+
+        if not new_user.created_by_partner:
+            send_welcome_email(new_user)
+
+        agent.record_custom_event("PartnerUserCreation", {"partner": self.partner.name})
+
+        return LinkResult(
+            user=new_user,
+            strategy=self.__class__.__name__,
+        )
+
+
+class ExistingUnlinkedUserStrategy(ClientMergeStrategy):
+    def process(self) -> LinkResult:
+        # IF it was scheduled to be deleted. Unschedule it.
+        self.user.delete_on = None
+        partner_user = ensure_partner_user_exists_for_user(
+            self.link_request, self.user, self.partner
+        )
+        set_plan_for_partner_user(partner_user, self.link_request.plan)
+
+        return LinkResult(
+            user=self.user,
+            strategy=self.__class__.__name__,
+        )
+
+
+class LinkedWithAnotherPartnerUserStrategy(ClientMergeStrategy):
+    def process(self) -> LinkResult:
+        raise AccountAlreadyLinkedToAnotherUserException()
+
+
+def get_login_strategy(
+    link_request: PartnerLinkRequest, user: Optional[User], partner: Partner
+) -> ClientMergeStrategy:
+    if user is None:
+        # We couldn't find any SimpleLogin user with the requested e-mail
+        return NewUserStrategy(link_request, user, partner)
+    # Check if user is already linked with another partner_user
+    other_partner_user = PartnerUser.get_by(partner_id=partner.id, user_id=user.id)
+    if other_partner_user is not None:
+        return LinkedWithAnotherPartnerUserStrategy(link_request, user, partner)
+    # There is a SimpleLogin user with the partner_user's e-mail
+    return ExistingUnlinkedUserStrategy(link_request, user, partner)
+
+
+def check_alias(email: str) -> bool:
+    alias = Alias.get_by(email=email)
+    if alias is not None:
+        raise AccountIsUsingAliasAsEmail()
+
+
+def process_login_case(
+    link_request: PartnerLinkRequest, partner: Partner
+) -> LinkResult:
+    # Sanitize email just in case
+    link_request.email = sanitize_email(link_request.email)
+    # Try to find a SimpleLogin user registered with that partner user id
+    partner_user = PartnerUser.get_by(
+        partner_id=partner.id, external_user_id=link_request.external_user_id
+    )
+    if partner_user is None:
+        canonical_email = canonicalize_email(link_request.email)
+        # We didn't find any SimpleLogin user registered with that partner user id
+        # Make sure they aren't using an alias as their link email
+        check_alias(link_request.email)
+        check_alias(canonical_email)
+        # Try to find it using the partner's e-mail address
+        users = User.filter(
+            or_(User.email == link_request.email, User.email == canonical_email)
+        ).all()
+        if len(users) > 1:
+            user = [user for user in users if user.email == canonical_email][0]
+        elif len(users) == 1:
+            user = users[0]
+        else:
+            user = None
+        return get_login_strategy(link_request, user, partner).process()
+    else:
+        # We found the SL user registered with that partner user id
+        # We're done
+        set_plan_for_partner_user(partner_user, link_request.plan)
+        # It's the same user. No need to do anything
+        return LinkResult(
+            user=partner_user.user,
+            strategy="Link",
+        )
+
+
+def link_user(
+    link_request: PartnerLinkRequest, current_user: User, partner: Partner
+) -> LinkResult:
+    # Sanitize email just in case
+    link_request.email = sanitize_email(link_request.email)
+    # If it was scheduled to be deleted. Unschedule it.
+    current_user.delete_on = None
+    partner_user = ensure_partner_user_exists_for_user(
+        link_request, current_user, partner
+    )
+    set_plan_for_partner_user(partner_user, link_request.plan)
+
+    agent.record_custom_event("AccountLinked", {"partner": partner.name})
+    Session.commit()
+    return LinkResult(
+        user=current_user,
+        strategy="Link",
+    )
+
+
+def switch_already_linked_user(
+    link_request: PartnerLinkRequest, partner_user: PartnerUser, current_user: User
+):
+    # Find if the user has another link and unlink it
+    other_partner_user = PartnerUser.get_by(
+        user_id=current_user.id,
+        partner_id=partner_user.partner_id,
+    )
+    if other_partner_user is not None:
+        LOG.i(
+            f"Deleting previous partner_user:{other_partner_user.id} from user:{current_user.id}"
+        )
+        PartnerUser.delete(other_partner_user.id)
+    LOG.i(f"Linking partner_user:{partner_user.id} to user:{current_user.id}")
+    # Link this partner_user to the current user
+    partner_user.user_id = current_user.id
+    # Set plan
+    set_plan_for_partner_user(partner_user, link_request.plan)
+    Session.commit()
+    return LinkResult(
+        user=current_user,
+        strategy="Link",
+    )
+
+
+def process_link_case(
+    link_request: PartnerLinkRequest,
+    current_user: User,
+    partner: Partner,
+) -> LinkResult:
+    # Sanitize email just in case
+    link_request.email = sanitize_email(link_request.email)
+    # Try to find a SimpleLogin user linked with this Partner account
+    partner_user = PartnerUser.get_by(
+        partner_id=partner.id, external_user_id=link_request.external_user_id
+    )
+    if partner_user is None:
+        # There is no SL user linked with the partner. Proceed with linking
+        return link_user(link_request, current_user, partner)
+
+    # There is a SL user registered with the partner. Check if is the current one
+    if partner_user.user_id == current_user.id:
+        # Update plan
+        set_plan_for_partner_user(partner_user, link_request.plan)
+        # It's the same user. No need to do anything
+        return LinkResult(
+            user=current_user,
+            strategy="Link",
+        )
+    else:
+        return switch_already_linked_user(link_request, partner_user, current_user)

app/admin_model.py

@@ -1,16 +1,89 @@
-from flask import redirect, url_for, request
+from typing import Optional
+
+import arrow
+import sqlalchemy
+from flask_admin.model.template import EndpointLinkRowAction
+from markupsafe import Markup
+
+from app import models, s3
+from flask import redirect, url_for, request, flash, Response
 from flask_admin import expose, AdminIndexView
+from flask_admin.actions import action
 from flask_admin.contrib import sqla
 from flask_login import current_user
 
+from app.db import Session
+from app.models import (
+    User,
+    ManualSubscription,
+    Fido,
+    Subscription,
+    AppleSubscription,
+    AdminAuditLog,
+    AuditLogActionEnum,
+    ProviderComplaintState,
+    Phase,
+    ProviderComplaint,
+    Alias,
+    Newsletter,
+    PADDLE_SUBSCRIPTION_GRACE_DAYS,
+)
+from app.newsletter_utils import send_newsletter_to_user, send_newsletter_to_address
+
 
 class SLModelView(sqla.ModelView):
+    column_default_sort = ("id", True)
+    column_display_pk = True
+    page_size = 100
+
+    can_edit = False
+    can_create = False
+    can_delete = False
+    edit_modal = True
+
     def is_accessible(self):
         return current_user.is_authenticated and current_user.is_admin
 
     def inaccessible_callback(self, name, **kwargs):
         # redirect to login page if user doesn't have access
-        return redirect(url_for("auth.login", next=request.url))
+        flash("You don't have access to the admin page", "error")
+        return redirect(url_for("dashboard.index", next=request.url))
+
+    def on_model_change(self, form, model, is_created):
+        changes = {}
+        for attr in sqlalchemy.inspect(model).attrs:
+            if attr.history.has_changes() and attr.key not in (
+                "created_at",
+                "updated_at",
+            ):
+                value = attr.value
+                # If it's a model reference, get the source id
+                if issubclass(type(value), models.Base):
+                    value = value.id
+                # otherwise, if its a generic object stringify it
+                if issubclass(type(value), object):
+                    value = str(value)
+                changes[attr.key] = value
+        auditAction = (
+            AuditLogActionEnum.create_object
+            if is_created
+            else AuditLogActionEnum.update_object
+        )
+        AdminAuditLog.create(
+            admin_user_id=current_user.id,
+            model=model.__class__.__name__,
+            model_id=model.id,
+            action=auditAction.value,
+            data=changes,
+        )
+
+    def on_model_delete(self, model):
+        AdminAuditLog.create(
+            admin_user_id=current_user.id,
+            model=model.__class__.__name__,
+            model_id=model.id,
+            action=AuditLogActionEnum.delete_object.value,
+        )
 
 
 class SLAdminIndexView(AdminIndexView):
@@ -19,4 +92,582 @@ class SLAdminIndexView(AdminIndexView):
         if not current_user.is_authenticated or not current_user.is_admin:
             return redirect(url_for("auth.login", next=request.url))
 
-        return super(SLAdminIndexView, self).index()
+        return redirect("/admin/user")
+
+
+def _user_upgrade_channel_formatter(view, context, model, name):
+    return Markup(model.upgrade_channel)
+
+
+class UserAdmin(SLModelView):
+    column_searchable_list = ["email", "id"]
+    column_exclude_list = [
+        "salt",
+        "password",
+        "otp_secret",
+        "last_otp",
+        "fido_uuid",
+        "profile_picture",
+    ]
+    can_edit = False
+
+    def scaffold_list_columns(self):
+        ret = super().scaffold_list_columns()
+        ret.insert(0, "upgrade_channel")
+        return ret
+
+    column_formatters = {
+        "upgrade_channel": _user_upgrade_channel_formatter,
+    }
+
+    @action(
+        "disable_user",
+        "Disable user",
+        "Are you sure you want to disable the selected users?",
+    )
+    def action_disable_user(self, ids):
+        for user in User.filter(User.id.in_(ids)):
+            user.disabled = True
+
+            flash(f"Disabled user {user.id}")
+            AdminAuditLog.disable_user(current_user.id, user.id)
+
+        Session.commit()
+
+    @action(
+        "enable_user",
+        "Enable user",
+        "Are you sure you want to enable the selected users?",
+    )
+    def action_enable_user(self, ids):
+        for user in User.filter(User.id.in_(ids)):
+            user.disabled = False
+
+            flash(f"Enabled user {user.id}")
+            AdminAuditLog.enable_user(current_user.id, user.id)
+
+        Session.commit()
+
+    @action(
+        "education_upgrade",
+        "Education upgrade",
+        "Are you sure you want to edu-upgrade selected users?",
+    )
+    def action_edu_upgrade(self, ids):
+        manual_upgrade("Edu", ids, is_giveaway=True)
+
+    @action(
+        "charity_org_upgrade",
+        "Charity Organization upgrade",
+        "Are you sure you want to upgrade selected users using the Charity organization program?",
+    )
+    def action_charity_org_upgrade(self, ids):
+        manual_upgrade("Charity Organization", ids, is_giveaway=True)
+
+    @action(
+        "journalist_upgrade",
+        "Journalist upgrade",
+        "Are you sure you want to upgrade selected users using the Journalist program?",
+    )
+    def action_journalist_upgrade(self, ids):
+        manual_upgrade("Journalist", ids, is_giveaway=True)
+
+    @action(
+        "cash_upgrade",
+        "Cash upgrade",
+        "Are you sure you want to cash-upgrade selected users?",
+    )
+    def action_cash_upgrade(self, ids):
+        manual_upgrade("Cash", ids, is_giveaway=False)
+
+    @action(
+        "crypto_upgrade",
+        "Crypto upgrade",
+        "Are you sure you want to crypto-upgrade selected users?",
+    )
+    def action_monero_upgrade(self, ids):
+        manual_upgrade("Crypto", ids, is_giveaway=False)
+
+    @action(
+        "adhoc_upgrade",
+        "Adhoc upgrade - for exceptional case",
+        "Are you sure you want to crypto-upgrade selected users?",
+    )
+    def action_adhoc_upgrade(self, ids):
+        manual_upgrade("Adhoc", ids, is_giveaway=False)
+
+    @action(
+        "extend_trial_1w",
+        "Extend trial for 1 week more",
+        "Extend trial for 1 week more?",
+    )
+    def extend_trial_1w(self, ids):
+        for user in User.filter(User.id.in_(ids)):
+            if user.trial_end and user.trial_end > arrow.now():
+                user.trial_end = user.trial_end.shift(weeks=1)
+            else:
+                user.trial_end = arrow.now().shift(weeks=1)
+
+            flash(f"Extend trial for {user} to {user.trial_end}", "success")
+            AdminAuditLog.extend_trial(
+                current_user.id, user.id, user.trial_end, "1 week"
+            )
+
+        Session.commit()
+
+    @action(
+        "remove trial",
+        "Stop trial period",
+        "Remove trial for this user?",
+    )
+    def stop_trial(self, ids):
+        for user in User.filter(User.id.in_(ids)):
+            user.trial_end = None
+
+            flash(f"Stopped trial for {user}", "success")
+            AdminAuditLog.stop_trial(current_user.id, user.id)
+
+        Session.commit()
+
+    @action(
+        "disable_otp_fido",
+        "Disable OTP & FIDO",
+        "Disable OTP & FIDO?",
+    )
+    def disable_otp_fido(self, ids):
+        for user in User.filter(User.id.in_(ids)):
+            user_had_otp = user.enable_otp
+            if user.enable_otp:
+                user.enable_otp = False
+                flash(f"Disable OTP for {user}", "info")
+
+            user_had_fido = user.fido_uuid is not None
+            if user.fido_uuid:
+                Fido.filter_by(uuid=user.fido_uuid).delete()
+                user.fido_uuid = None
+                flash(f"Disable FIDO for {user}", "info")
+            AdminAuditLog.disable_otp_fido(
+                current_user.id, user.id, user_had_otp, user_had_fido
+            )
+
+        Session.commit()
+
+    @action(
+        "stop_paddle_sub",
+        "Stop user Paddle subscription",
+        "This will stop the current user Paddle subscription so if user doesn't have Proton sub, they will lose all SL benefits immediately",
+    )
+    def stop_paddle_sub(self, ids):
+        for user in User.filter(User.id.in_(ids)):
+            sub: Subscription = user.get_paddle_subscription()
+            if not sub:
+                flash(f"No Paddle sub for {user}", "warning")
+                continue
+
+            flash(f"{user} sub will end now, instead of {sub.next_bill_date}", "info")
+            sub.next_bill_date = (
+                arrow.now().shift(days=-PADDLE_SUBSCRIPTION_GRACE_DAYS).date()
+            )
+
+        Session.commit()
+
+    @action(
+        "clear_delete_on",
+        "Remove scheduled deletion of user",
+        "This will remove the scheduled deletion for this users",
+    )
+    def clean_delete_on(self, ids):
+        for user in User.filter(User.id.in_(ids)):
+            user.delete_on = None
+
+        Session.commit()
+
+    # @action(
+    #     "login_as",
+    #     "Login as this user",
+    #     "Login as this user?",
+    # )
+    # def login_as(self, ids):
+    #     if len(ids) != 1:
+    #         flash("only 1 user can be selected", "error")
+    #         return
+    #
+    #     for user in User.filter(User.id.in_(ids)):
+    #         AdminAuditLog.logged_as_user(current_user.id, user.id)
+    #         login_user(user)
+    #         flash(f"Login as user {user}", "success")
+    #         return redirect("/")
+
+
+def manual_upgrade(way: str, ids: [int], is_giveaway: bool):
+    for user in User.filter(User.id.in_(ids)).all():
+        if user.lifetime:
+            flash(f"user {user} already has a lifetime license", "warning")
+            continue
+
+        sub: Subscription = user.get_paddle_subscription()
+        if sub and not sub.cancelled:
+            flash(
+                f"user {user} already has a Paddle license, they have to cancel it first",
+                "warning",
+            )
+            continue
+
+        apple_sub: AppleSubscription = AppleSubscription.get_by(user_id=user.id)
+        if apple_sub and apple_sub.is_valid():
+            flash(
+                f"user {user} already has a Apple subscription, they have to cancel it first",
+                "warning",
+            )
+            continue
+
+        AdminAuditLog.create_manual_upgrade(current_user.id, way, user.id, is_giveaway)
+        manual_sub: ManualSubscription = ManualSubscription.get_by(user_id=user.id)
+        if manual_sub:
+            # renew existing subscription
+            if manual_sub.end_at > arrow.now():
+                manual_sub.end_at = manual_sub.end_at.shift(years=1)
+            else:
+                manual_sub.end_at = arrow.now().shift(years=1, days=1)
+            flash(f"Subscription extended to {manual_sub.end_at.humanize()}", "success")
+            continue
+
+        ManualSubscription.create(
+            user_id=user.id,
+            end_at=arrow.now().shift(years=1, days=1),
+            comment=way,
+            is_giveaway=is_giveaway,
+        )
+
+        flash(f"New {way} manual subscription for {user} is created", "success")
+    Session.commit()
+
+
+class EmailLogAdmin(SLModelView):
+    column_searchable_list = ["id"]
+    column_filters = ["id", "user.email", "mailbox.email", "contact.website_email"]
+
+    can_edit = False
+    can_create = False
+
+
+class AliasAdmin(SLModelView):
+    column_searchable_list = ["id", "user.email", "email", "mailbox.email"]
+    column_filters = ["id", "user.email", "email", "mailbox.email"]
+
+    @action(
+        "disable_email_spoofing_check",
+        "Disable email spoofing protection",
+        "Disable email spoofing protection?",
+    )
+    def disable_email_spoofing_check_for(self, ids):
+        for alias in Alias.filter(Alias.id.in_(ids)):
+            if alias.disable_email_spoofing_check:
+                flash(
+                    f"Email spoofing protection is already disabled on {alias.email}",
+                    "warning",
+                )
+            else:
+                alias.disable_email_spoofing_check = True
+                flash(
+                    f"Email spoofing protection is disabled on {alias.email}", "success"
+                )
+
+        Session.commit()
+
+
+class MailboxAdmin(SLModelView):
+    column_searchable_list = ["id", "user.email", "email"]
+    column_filters = ["id", "user.email", "email"]
+
+
+# class LifetimeCouponAdmin(SLModelView):
+#     can_edit = True
+#     can_create = True
+
+
+class CouponAdmin(SLModelView):
+    can_edit = False
+    can_create = True
+
+
+class ManualSubscriptionAdmin(SLModelView):
+    can_edit = True
+    column_searchable_list = ["id", "user.email"]
+
+    @action(
+        "extend_1y",
+        "Extend for 1 year",
+        "Extend 1 year more?",
+    )
+    def extend_1y(self, ids):
+        for ms in ManualSubscription.filter(ManualSubscription.id.in_(ids)):
+            ms.end_at = ms.end_at.shift(years=1)
+            flash(f"Extend subscription for 1 year for {ms.user}", "success")
+            AdminAuditLog.extend_subscription(
+                current_user.id, ms.user.id, ms.end_at, "1 year"
+            )
+
+        Session.commit()
+
+    @action(
+        "extend_1m",
+        "Extend for 1 month",
+        "Extend 1 month more?",
+    )
+    def extend_1m(self, ids):
+        for ms in ManualSubscription.filter(ManualSubscription.id.in_(ids)):
+            ms.end_at = ms.end_at.shift(months=1)
+            flash(f"Extend subscription for 1 month for {ms.user}", "success")
+            AdminAuditLog.extend_subscription(
+                current_user.id, ms.user.id, ms.end_at, "1 month"
+            )
+
+        Session.commit()
+
+
+# class ClientAdmin(SLModelView):
+#     column_searchable_list = ["name", "description", "user.email"]
+#     column_exclude_list = ["oauth_client_secret", "home_url"]
+#     can_edit = True
+
+
+class CustomDomainAdmin(SLModelView):
+    column_searchable_list = ["domain", "user.email", "user.id"]
+    column_exclude_list = ["ownership_txt_token"]
+    can_edit = False
+
+
+class ReferralAdmin(SLModelView):
+    column_searchable_list = ["id", "user.email", "code", "name"]
+    column_filters = ["id", "user.email", "code", "name"]
+
+    def scaffold_list_columns(self):
+        ret = super().scaffold_list_columns()
+        ret.insert(0, "nb_user")
+        ret.insert(0, "nb_paid_user")
+        return ret
+
+
+# class PayoutAdmin(SLModelView):
+#     column_searchable_list = ["id", "user.email"]
+#     column_filters = ["id", "user.email"]
+#     can_edit = True
+#     can_create = True
+#     can_delete = True
+
+
+def _admin_action_formatter(view, context, model, name):
+    action_name = AuditLogActionEnum.get_name(model.action)
+    return "{} ({})".format(action_name, model.action)
+
+
+def _admin_created_at_formatter(view, context, model, name):
+    return model.created_at.format()
+
+
+class AdminAuditLogAdmin(SLModelView):
+    column_searchable_list = ["admin.id", "admin.email", "model_id", "created_at"]
+    column_filters = ["admin.id", "admin.email", "model_id", "created_at"]
+    column_exclude_list = ["id"]
+    column_hide_backrefs = False
+    can_edit = False
+    can_create = False
+    can_delete = False
+
+    column_formatters = {
+        "action": _admin_action_formatter,
+        "created_at": _admin_created_at_formatter,
+    }
+
+
+def _transactionalcomplaint_state_formatter(view, context, model, name):
+    return "{} ({})".format(ProviderComplaintState(model.state).name, model.state)
+
+
+def _transactionalcomplaint_phase_formatter(view, context, model, name):
+    return Phase(model.phase).name
+
+
+def _transactionalcomplaint_refused_email_id_formatter(view, context, model, name):
+    markupstring = "<a href='{}'>{}</a>".format(
+        url_for(".download_eml", id=model.id), model.refused_email.full_report_path
+    )
+    return Markup(markupstring)
+
+
+class ProviderComplaintAdmin(SLModelView):
+    column_searchable_list = ["id", "user.id", "created_at"]
+    column_filters = ["user.id", "state"]
+    column_hide_backrefs = False
+    can_edit = False
+    can_create = False
+    can_delete = False
+
+    column_formatters = {
+        "created_at": _admin_created_at_formatter,
+        "updated_at": _admin_created_at_formatter,
+        "state": _transactionalcomplaint_state_formatter,
+        "phase": _transactionalcomplaint_phase_formatter,
+        "refused_email": _transactionalcomplaint_refused_email_id_formatter,
+    }
+
+    column_extra_row_actions = [  # Add a new action button
+        EndpointLinkRowAction("fa fa-check-square", ".mark_ok"),
+    ]
+
+    def _get_complaint(self) -> Optional[ProviderComplaint]:
+        complain_id = request.args.get("id")
+        if complain_id is None:
+            flash("Missing id", "error")
+            return None
+        complaint = ProviderComplaint.get_by(id=complain_id)
+        if not complaint:
+            flash("Could not find complaint", "error")
+            return None
+        return complaint
+
+    @expose("/mark_ok", methods=["GET"])
+    def mark_ok(self):
+        complaint = self._get_complaint()
+        if not complaint:
+            return redirect("/admin/transactionalcomplaint/")
+        complaint.state = ProviderComplaintState.reviewed.value
+        Session.commit()
+        return redirect("/admin/transactionalcomplaint/")
+
+    @expose("/download_eml", methods=["GET"])
+    def download_eml(self):
+        complaint = self._get_complaint()
+        if not complaint:
+            return redirect("/admin/transactionalcomplaint/")
+        eml_path = complaint.refused_email.full_report_path
+        eml_data = s3.download_email(eml_path)
+        AdminAuditLog.downloaded_provider_complaint(current_user.id, complaint.id)
+        Session.commit()
+        return Response(
+            eml_data,
+            mimetype="message/rfc822",
+            headers={
+                "Content-Disposition": "attachment;filename={}".format(
+                    complaint.refused_email.path
+                )
+            },
+        )
+
+
+def _newsletter_plain_text_formatter(view, context, model: Newsletter, name):
+    # to display newsletter plain_text with linebreaks in the list view
+    return Markup(model.plain_text.replace("\n", "<br>"))
+
+
+def _newsletter_html_formatter(view, context, model: Newsletter, name):
+    # to display newsletter html with linebreaks in the list view
+    return Markup(model.html.replace("\n", "<br>"))
+
+
+class NewsletterAdmin(SLModelView):
+    list_template = "admin/model/newsletter-list.html"
+    edit_template = "admin/model/newsletter-edit.html"
+    edit_modal = False
+
+    can_edit = True
+    can_create = True
+
+    column_formatters = {
+        "plain_text": _newsletter_plain_text_formatter,
+        "html": _newsletter_html_formatter,
+    }
+
+    @action(
+        "send_newsletter_to_user",
+        "Send this newsletter to myself or the specified userID",
+    )
+    def send_newsletter_to_user(self, newsletter_ids):
+        user_id = request.form["user_id"]
+        if user_id:
+            user = User.get(user_id)
+            if not user:
+                flash(f"No such user with ID {user_id}", "error")
+                return
+        else:
+            flash("use the current user", "info")
+            user = current_user
+
+        for newsletter_id in newsletter_ids:
+            newsletter = Newsletter.get(newsletter_id)
+            sent, error_msg = send_newsletter_to_user(newsletter, user)
+            if sent:
+                flash(f"{newsletter} sent to {user}", "success")
+            else:
+                flash(error_msg, "error")
+
+    @action(
+        "send_newsletter_to_address",
+        "Send this newsletter to a specific address",
+    )
+    def send_newsletter_to_address(self, newsletter_ids):
+        to_address = request.form["to_address"]
+        if not to_address:
+            flash("to_address missing", "error")
+            return
+
+        for newsletter_id in newsletter_ids:
+            newsletter = Newsletter.get(newsletter_id)
+            # use the current_user for rendering email
+            sent, error_msg = send_newsletter_to_address(
+                newsletter, current_user, to_address
+            )
+            if sent:
+                flash(
+                    f"{newsletter} sent to {to_address} with {current_user} context",
+                    "success",
+                )
+            else:
+                flash(error_msg, "error")
+
+    @action(
+        "clone_newsletter",
+        "Clone this newsletter",
+    )
+    def clone_newsletter(self, newsletter_ids):
+        if len(newsletter_ids) != 1:
+            flash("you can only select 1 newsletter", "error")
+            return
+
+        newsletter_id = newsletter_ids[0]
+        newsletter: Newsletter = Newsletter.get(newsletter_id)
+        new_newsletter = Newsletter.create(
+            subject=newsletter.subject,
+            html=newsletter.html,
+            plain_text=newsletter.plain_text,
+            commit=True,
+        )
+
+        flash(f"Newsletter {new_newsletter.subject} has been cloned", "success")
+
+
+class NewsletterUserAdmin(SLModelView):
+    column_searchable_list = ["id"]
+    column_filters = ["id", "user.email", "newsletter.subject"]
+    column_exclude_list = ["created_at", "updated_at", "id"]
+
+    can_edit = False
+    can_create = False
+
+
+class DailyMetricAdmin(SLModelView):
+    column_exclude_list = ["created_at", "updated_at", "id"]
+
+    can_export = True
+
+
+class MetricAdmin(SLModelView):
+    column_exclude_list = ["created_at", "updated_at", "id"]
+
+    can_export = True
+
+
+class InvalidMailboxDomainAdmin(SLModelView):
+    can_create = True
+    can_delete = True

app/alias_suffix.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+import json
+from dataclasses import asdict, dataclass
+from typing import Optional
+
+import itsdangerous
+from app import config
+from app.log import LOG
+from app.models import User, AliasOptions, SLDomain
+
+signer = itsdangerous.TimestampSigner(config.CUSTOM_ALIAS_SECRET)
+
+
+@dataclass
+class AliasSuffix:
+    # whether this is a custom domain
+    is_custom: bool
+    # Suffix
+    suffix: str
+    # Suffix signature
+    signed_suffix: str
+    # whether this is a premium SL domain. Not apply to custom domain
+    is_premium: bool
+    # can be either Custom or SL domain
+    domain: str
+    # if custom domain, whether the custom domain has MX verified, i.e. can receive emails
+    mx_verified: bool = True
+
+    def serialize(self):
+        return json.dumps(asdict(self))
+
+    @classmethod
+    def deserialize(cls, data: str) -> AliasSuffix:
+        return AliasSuffix(**json.loads(data))
+
+
+def check_suffix_signature(signed_suffix: str) -> Optional[str]:
+    # hypothesis: user will click on the button in the 600 secs
+    try:
+        return signer.unsign(signed_suffix, max_age=600).decode()
+    except itsdangerous.BadSignature:
+        return None
+
+
+def verify_prefix_suffix(
+    user: User, alias_prefix, alias_suffix, alias_options: Optional[AliasOptions] = None
+) -> bool:
+    """verify if user could create an alias with the given prefix and suffix"""
+    if not alias_prefix or not alias_suffix:  # should be caught on frontend
+        return False
+
+    user_custom_domains = [cd.domain for cd in user.verified_custom_domains()]
+
+    # make sure alias_suffix is either .random_word@simplelogin.co or @my-domain.com
+    alias_suffix = alias_suffix.strip()
+    # alias_domain_prefix is either a .random_word or ""
+    alias_domain_prefix, alias_domain = alias_suffix.split("@", 1)
+
+    # alias_domain must be either one of user custom domains or built-in domains
+    if alias_domain not in user.available_alias_domains(alias_options=alias_options):
+        LOG.e("wrong alias suffix %s, user %s", alias_suffix, user)
+        return False
+
+    # SimpleLogin domain case:
+    # 1) alias_suffix must start with "." and
+    # 2) alias_domain_prefix must come from the word list
+    if (
+        alias_domain in user.available_sl_domains(alias_options=alias_options)
+        and alias_domain not in user_custom_domains
+        # when DISABLE_ALIAS_SUFFIX is true, alias_domain_prefix is empty
+        and not config.DISABLE_ALIAS_SUFFIX
+    ):
+        if not alias_domain_prefix.startswith("."):
+            LOG.e("User %s submits a wrong alias suffix %s", user, alias_suffix)
+            return False
+
+    else:
+        if alias_domain not in user_custom_domains:
+            if not config.DISABLE_ALIAS_SUFFIX:
+                LOG.e("wrong alias suffix %s, user %s", alias_suffix, user)
+                return False
+
+            if alias_domain not in user.available_sl_domains(
+                alias_options=alias_options
+            ):
+                LOG.e("wrong alias suffix %s, user %s", alias_suffix, user)
+                return False
+
+    return True
+
+
+def get_alias_suffixes(
+    user: User, alias_options: Optional[AliasOptions] = None
+) -> [AliasSuffix]:
+    """
+    Similar to as get_available_suffixes() but also return custom domain that doesn't have MX set up.
+    """
+    user_custom_domains = user.verified_custom_domains()
+
+    alias_suffixes: [AliasSuffix] = []
+
+    # put custom domain first
+    # for each user domain, generate both the domain and a random suffix version
+    for custom_domain in user_custom_domains:
+        if custom_domain.random_prefix_generation:
+            suffix = (
+                f".{user.get_random_alias_suffix(custom_domain)}@{custom_domain.domain}"
+            )
+            alias_suffix = AliasSuffix(
+                is_custom=True,
+                suffix=suffix,
+                signed_suffix=signer.sign(suffix).decode(),
+                is_premium=False,
+                domain=custom_domain.domain,
+                mx_verified=custom_domain.verified,
+            )
+            if user.default_alias_custom_domain_id == custom_domain.id:
+                alias_suffixes.insert(0, alias_suffix)
+            else:
+                alias_suffixes.append(alias_suffix)
+
+        suffix = f"@{custom_domain.domain}"
+        alias_suffix = AliasSuffix(
+            is_custom=True,
+            suffix=suffix,
+            signed_suffix=signer.sign(suffix).decode(),
+            is_premium=False,
+            domain=custom_domain.domain,
+            mx_verified=custom_domain.verified,
+        )
+
+        # put the default domain to top
+        # only if random_prefix_generation isn't enabled
+        if (
+            user.default_alias_custom_domain_id == custom_domain.id
+            and not custom_domain.random_prefix_generation
+        ):
+            alias_suffixes.insert(0, alias_suffix)
+        else:
+            alias_suffixes.append(alias_suffix)
+
+    # then SimpleLogin domain
+    sl_domains = user.get_sl_domains(alias_options=alias_options)
+    default_domain_found = False
+    for sl_domain in sl_domains:
+        prefix = (
+            "" if config.DISABLE_ALIAS_SUFFIX else f".{user.get_random_alias_suffix()}"
+        )
+        suffix = f"{prefix}@{sl_domain.domain}"
+        alias_suffix = AliasSuffix(
+            is_custom=False,
+            suffix=suffix,
+            signed_suffix=signer.sign(suffix).decode(),
+            is_premium=sl_domain.premium_only,
+            domain=sl_domain.domain,
+            mx_verified=True,
+        )
+        # No default or this is not the default
+        if (
+            user.default_alias_public_domain_id is None
+            or user.default_alias_public_domain_id != sl_domain.id
+        ):
+            alias_suffixes.append(alias_suffix)
+        else:
+            default_domain_found = True
+            alias_suffixes.insert(0, alias_suffix)
+
+    if not default_domain_found:
+        domain_conditions = {"id": user.default_alias_public_domain_id, "hidden": False}
+        if not user.is_premium():
+            domain_conditions["premium_only"] = False
+        sl_domain = SLDomain.get_by(**domain_conditions)
+        if sl_domain:
+            prefix = (
+                ""
+                if config.DISABLE_ALIAS_SUFFIX
+                else f".{user.get_random_alias_suffix()}"
+            )
+            suffix = f"{prefix}@{sl_domain.domain}"
+            alias_suffix = AliasSuffix(
+                is_custom=False,
+                suffix=suffix,
+                signed_suffix=signer.sign(suffix).decode(),
+                is_premium=sl_domain.premium_only,
+                domain=sl_domain.domain,
+                mx_verified=True,
+            )
+            alias_suffixes.insert(0, alias_suffix)
+
+    return alias_suffixes

app/alias_utils.py

@@ -0,0 +1,479 @@
+import csv
+from io import StringIO
+import re
+from typing import Optional, Tuple
+
+from email_validator import validate_email, EmailNotValidError
+from sqlalchemy.exc import IntegrityError, DataError
+from flask import make_response
+
+from app.config import (
+    BOUNCE_PREFIX_FOR_REPLY_PHASE,
+    BOUNCE_PREFIX,
+    BOUNCE_SUFFIX,
+    VERP_PREFIX,
+)
+from app.db import Session
+from app.email_utils import (
+    get_email_domain_part,
+    send_cannot_create_directory_alias,
+    can_create_directory_for_address,
+    send_cannot_create_directory_alias_disabled,
+    get_email_local_part,
+    send_cannot_create_domain_alias,
+    send_email,
+    render,
+)
+from app.errors import AliasInTrashError
+from app.events.event_dispatcher import EventDispatcher
+from app.events.generated.event_pb2 import AliasDeleted, AliasStatusChange, EventContent
+from app.log import LOG
+from app.models import (
+    Alias,
+    CustomDomain,
+    Directory,
+    User,
+    DeletedAlias,
+    DomainDeletedAlias,
+    AliasMailbox,
+    Mailbox,
+    EmailLog,
+    Contact,
+    AutoCreateRule,
+    AliasUsedOn,
+    ClientUser,
+)
+from app.regex_utils import regex_match
+
+
+def get_user_if_alias_would_auto_create(
+    address: str, notify_user: bool = False
+) -> Optional[User]:
+    banned_prefix = f"{VERP_PREFIX}."
+    if address.startswith(banned_prefix):
+        LOG.w("alias %s can't start with %s", address, banned_prefix)
+        return None
+
+    try:
+        # Prevent addresses with unicode characters (🤯) in them for now.
+        validate_email(address, check_deliverability=False, allow_smtputf8=False)
+    except EmailNotValidError:
+        return None
+
+    domain_and_rule = check_if_alias_can_be_auto_created_for_custom_domain(
+        address, notify_user=notify_user
+    )
+    if DomainDeletedAlias.get_by(email=address):
+        return None
+    if domain_and_rule:
+        return domain_and_rule[0].user
+    directory = check_if_alias_can_be_auto_created_for_a_directory(
+        address, notify_user=notify_user
+    )
+    if directory:
+        return directory.user
+
+    return None
+
+
+def check_if_alias_can_be_auto_created_for_custom_domain(
+    address: str, notify_user: bool = True
+) -> Optional[Tuple[CustomDomain, Optional[AutoCreateRule]]]:
+    """
+    Check if this address would generate an auto created alias.
+    If that's the case return the domain that would create it and the rule that triggered it.
+    If there's no rule it's a catchall creation
+    """
+    alias_domain = get_email_domain_part(address)
+    custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain)
+
+    if not custom_domain:
+        return None
+
+    user: User = custom_domain.user
+    if user.disabled:
+        LOG.i("Disabled user %s can't create new alias via custom domain", user)
+        return None
+
+    if not user.can_create_new_alias():
+        LOG.d(f"{user} can't create new custom-domain alias {address}")
+        if notify_user:
+            send_cannot_create_domain_alias(custom_domain.user, address, alias_domain)
+        return None
+
+    if not custom_domain.catch_all:
+        if len(custom_domain.auto_create_rules) == 0:
+            return None
+        local = get_email_local_part(address)
+
+        for rule in custom_domain.auto_create_rules:
+            if regex_match(rule.regex, local):
+                LOG.d(
+                    "%s passes %s on %s",
+                    address,
+                    rule.regex,
+                    custom_domain,
+                )
+                return custom_domain, rule
+        else:  # no rule passes
+            LOG.d("no rule passed to create %s", local)
+            return None
+    LOG.d("Create alias via catchall")
+
+    return custom_domain, None
+
+
+def check_if_alias_can_be_auto_created_for_a_directory(
+    address: str, notify_user: bool = True
+) -> Optional[Directory]:
+    """
+    Try to create an alias with directory
+    If an alias would be created, return the dictionary that would trigger the creation. Otherwise, return None.
+    """
+    # check if alias belongs to a directory, ie having directory/anything@EMAIL_DOMAIN format
+    if not can_create_directory_for_address(address):
+        return None
+
+    # alias contains one of the 3 special directory separator: "/", "+" or "#"
+    if "/" in address:
+        sep = "/"
+    elif "+" in address:
+        sep = "+"
+    elif "#" in address:
+        sep = "#"
+    else:
+        # if there's no directory separator in the alias, no way to auto-create it
+        return None
+
+    directory_name = address[: address.find(sep)]
+    LOG.d("directory_name %s", directory_name)
+
+    directory = Directory.get_by(name=directory_name)
+    if not directory:
+        return None
+
+    user: User = directory.user
+    if user.disabled:
+        LOG.i("Disabled %s can't create new alias with directory", user)
+        return None
+
+    if not user.can_create_new_alias():
+        LOG.d(f"{user} can't create new directory alias {address}")
+        if notify_user:
+            send_cannot_create_directory_alias(user, address, directory_name)
+        return None
+
+    if directory.disabled:
+        if notify_user:
+            send_cannot_create_directory_alias_disabled(user, address, directory_name)
+        return None
+
+    return directory
+
+
+def try_auto_create(address: str) -> Optional[Alias]:
+    """Try to auto-create the alias using directory or catch-all domain"""
+    # VERP for reply phase is {BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}
+    if address.startswith(f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+") and "+@" in address:
+        LOG.e("alias %s can't start with %s", address, BOUNCE_PREFIX_FOR_REPLY_PHASE)
+        return None
+
+    # VERP for forward phase is BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX
+    if address.startswith(BOUNCE_PREFIX) and address.endswith(BOUNCE_SUFFIX):
+        LOG.e("alias %s can't start with %s", address, BOUNCE_PREFIX)
+        return None
+
+    try:
+        # NOT allow unicode for now
+        validate_email(address, check_deliverability=False, allow_smtputf8=False)
+    except EmailNotValidError:
+        return None
+
+    alias = try_auto_create_via_domain(address)
+    if not alias:
+        alias = try_auto_create_directory(address)
+
+    return alias
+
+
+def try_auto_create_directory(address: str) -> Optional[Alias]:
+    """
+    Try to create an alias with directory
+    """
+    directory = check_if_alias_can_be_auto_created_for_a_directory(
+        address, notify_user=True
+    )
+    if not directory:
+        return None
+
+    try:
+        LOG.d("create alias %s for directory %s", address, directory)
+
+        mailboxes = directory.mailboxes
+
+        alias = Alias.create(
+            email=address,
+            user_id=directory.user_id,
+            directory_id=directory.id,
+            mailbox_id=mailboxes[0].id,
+        )
+        if not directory.user.disable_automatic_alias_note:
+            alias.note = f"Created by directory {directory.name}"
+        Session.flush()
+        for i in range(1, len(mailboxes)):
+            AliasMailbox.create(
+                alias_id=alias.id,
+                mailbox_id=mailboxes[i].id,
+            )
+
+        Session.commit()
+        return alias
+    except AliasInTrashError:
+        LOG.w(
+            "Alias %s was deleted before, cannot auto-create using directory %s, user %s",
+            address,
+            directory.name,
+            directory.user,
+        )
+        return None
+    except IntegrityError:
+        LOG.w("Alias %s already exists", address)
+        Session.rollback()
+        alias = Alias.get_by(email=address)
+        return alias
+
+
+def try_auto_create_via_domain(address: str) -> Optional[Alias]:
+    """Try to create an alias with catch-all or auto-create rules on custom domain"""
+    can_create = check_if_alias_can_be_auto_created_for_custom_domain(address)
+    if not can_create:
+        return None
+    custom_domain, rule = can_create
+
+    if rule:
+        alias_note = f"Created by rule {rule.order} with regex {rule.regex}"
+        mailboxes = rule.mailboxes
+    else:
+        alias_note = "Created by catchall option"
+        mailboxes = custom_domain.mailboxes
+
+    # a rule can have 0 mailboxes. Happened when a mailbox is deleted
+    if not mailboxes:
+        LOG.d(
+            "use %s default mailbox for %s %s",
+            custom_domain.user,
+            address,
+            custom_domain,
+        )
+        mailboxes = [custom_domain.user.default_mailbox]
+
+    try:
+        LOG.d("create alias %s for domain %s", address, custom_domain)
+        alias = Alias.create(
+            email=address,
+            user_id=custom_domain.user_id,
+            custom_domain_id=custom_domain.id,
+            automatic_creation=True,
+            mailbox_id=mailboxes[0].id,
+        )
+        if not custom_domain.user.disable_automatic_alias_note:
+            alias.note = alias_note
+        Session.flush()
+        for i in range(1, len(mailboxes)):
+            AliasMailbox.create(
+                alias_id=alias.id,
+                mailbox_id=mailboxes[i].id,
+            )
+        Session.commit()
+        return alias
+    except AliasInTrashError:
+        LOG.w(
+            "Alias %s was deleted before, cannot auto-create using domain catch-all %s, user %s",
+            address,
+            custom_domain,
+            custom_domain.user,
+        )
+        return None
+    except IntegrityError:
+        LOG.w("Alias %s already exists", address)
+        Session.rollback()
+        alias = Alias.get_by(email=address)
+        return alias
+    except DataError:
+        LOG.w("Cannot create alias %s", address)
+        Session.rollback()
+        return None
+
+
+def delete_alias(alias: Alias, user: User):
+    """
+    Delete an alias and add it to either global or domain trash
+    Should be used instead of Alias.delete, DomainDeletedAlias.create, DeletedAlias.create
+    """
+    LOG.i(f"User {user} has deleted alias {alias}")
+    # save deleted alias to either global or domain tra
+    if alias.custom_domain_id:
+        if not DomainDeletedAlias.get_by(
+            email=alias.email, domain_id=alias.custom_domain_id
+        ):
+            domain_deleted_alias = DomainDeletedAlias(
+                user_id=user.id,
+                email=alias.email,
+                domain_id=alias.custom_domain_id,
+            )
+            Session.add(domain_deleted_alias)
+            Session.commit()
+            LOG.i(
+                f"Moving {alias} to domain {alias.custom_domain_id} trash {domain_deleted_alias}"
+            )
+    else:
+        if not DeletedAlias.get_by(email=alias.email):
+            deleted_alias = DeletedAlias(email=alias.email)
+            Session.add(deleted_alias)
+            Session.commit()
+            LOG.i(f"Moving {alias} to global trash {deleted_alias}")
+
+    Alias.filter(Alias.id == alias.id).delete()
+    Session.commit()
+
+    EventDispatcher.send_event(
+        user, EventContent(alias_deleted=AliasDeleted(alias_id=alias.id))
+    )
+
+
+def aliases_for_mailbox(mailbox: Mailbox) -> [Alias]:
+    """
+    get list of aliases for a given mailbox
+    """
+    ret = set(Alias.filter(Alias.mailbox_id == mailbox.id).all())
+
+    for alias in (
+        Session.query(Alias)
+        .join(AliasMailbox, Alias.id == AliasMailbox.alias_id)
+        .filter(AliasMailbox.mailbox_id == mailbox.id)
+    ):
+        ret.add(alias)
+
+    return list(ret)
+
+
+def nb_email_log_for_mailbox(mailbox: Mailbox):
+    aliases = aliases_for_mailbox(mailbox)
+    alias_ids = [alias.id for alias in aliases]
+    return (
+        Session.query(EmailLog)
+        .join(Contact, EmailLog.contact_id == Contact.id)
+        .filter(Contact.alias_id.in_(alias_ids))
+        .count()
+    )
+
+
+# Only lowercase letters, numbers, dots (.), dashes (-) and underscores (_) are currently supported
+_ALIAS_PREFIX_PATTERN = r"[0-9a-z-_.]{1,}"
+
+
+def check_alias_prefix(alias_prefix) -> bool:
+    if len(alias_prefix) > 40:
+        return False
+
+    if re.fullmatch(_ALIAS_PREFIX_PATTERN, alias_prefix) is None:
+        return False
+
+    return True
+
+
+def alias_export_csv(user, csv_direct_export=False):
+    """
+    Get user aliases as importable CSV file
+    Output:
+        Importable CSV file
+
+    """
+    data = [["alias", "note", "enabled", "mailboxes"]]
+    for alias in Alias.filter_by(user_id=user.id).all():  # type: Alias
+        # Always put the main mailbox first
+        # It is seen a primary while importing
+        alias_mailboxes = alias.mailboxes
+        alias_mailboxes.insert(
+            0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox))
+        )
+
+        mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes])
+        data.append([alias.email, alias.note, alias.enabled, mailboxes])
+
+    si = StringIO()
+    cw = csv.writer(si)
+    cw.writerows(data)
+    if csv_direct_export:
+        return si.getvalue()
+    output = make_response(si.getvalue())
+    output.headers["Content-Disposition"] = "attachment; filename=aliases.csv"
+    output.headers["Content-type"] = "text/csv"
+    return output
+
+
+def transfer_alias(alias, new_user, new_mailboxes: [Mailbox]):
+    # cannot transfer alias which is used for receiving newsletter
+    if User.get_by(newsletter_alias_id=alias.id):
+        raise Exception("Cannot transfer alias that's used to receive newsletter")
+
+    # update user_id
+    Session.query(Contact).filter(Contact.alias_id == alias.id).update(
+        {"user_id": new_user.id}
+    )
+
+    Session.query(AliasUsedOn).filter(AliasUsedOn.alias_id == alias.id).update(
+        {"user_id": new_user.id}
+    )
+
+    Session.query(ClientUser).filter(ClientUser.alias_id == alias.id).update(
+        {"user_id": new_user.id}
+    )
+
+    # remove existing mailboxes from the alias
+    Session.query(AliasMailbox).filter(AliasMailbox.alias_id == alias.id).delete()
+
+    # set mailboxes
+    alias.mailbox_id = new_mailboxes.pop().id
+    for mb in new_mailboxes:
+        AliasMailbox.create(alias_id=alias.id, mailbox_id=mb.id)
+
+    # alias has never been transferred before
+    if not alias.original_owner_id:
+        alias.original_owner_id = alias.user_id
+
+    # inform previous owner
+    old_user = alias.user
+    send_email(
+        old_user.email,
+        f"Alias {alias.email} has been received",
+        render(
+            "transactional/alias-transferred.txt",
+            alias=alias,
+        ),
+        render(
+            "transactional/alias-transferred.html",
+            alias=alias,
+        ),
+    )
+
+    # now the alias belongs to the new user
+    alias.user_id = new_user.id
+
+    # set some fields back to default
+    alias.disable_pgp = False
+    alias.pinned = False
+
+    Session.commit()
+
+
+def change_alias_status(alias: Alias, enabled: bool, commit: bool = False):
+    alias.enabled = enabled
+
+    event = AliasStatusChange(
+        alias_id=alias.id, alias_email=alias.email, enabled=enabled
+    )
+    EventDispatcher.send_event(alias.user, EventContent(alias_status_change=event))
+
+    if commit:
+        Session.commit()

app/api/__init__.py

@@ -1,9 +1,37 @@
 from .views import (
     alias_options,
     new_custom_alias,
+    custom_domain,
     new_random_alias,
     user_info,
-    auth_login,
+    auth,
     auth_mfa,
     alias,
+    apple,
+    mailbox,
+    notification,
+    setting,
+    export,
+    phone,
+    sudo,
+    user,
 )
+
+__all__ = [
+    "alias_options",
+    "new_custom_alias",
+    "custom_domain",
+    "new_random_alias",
+    "user_info",
+    "auth",
+    "auth_mfa",
+    "alias",
+    "apple",
+    "mailbox",
+    "notification",
+    "setting",
+    "export",
+    "phone",
+    "sudo",
+    "user",
+]

app/api/base.py

@@ -1,30 +1,70 @@
 from functools import wraps
+from typing import Tuple, Optional
 
 import arrow
 from flask import Blueprint, request, jsonify, g
+from flask_login import current_user
 
-from app.extensions import db
+from app.db import Session
 from app.models import ApiKey
 
 api_bp = Blueprint(name="api", import_name=__name__, url_prefix="/api")
 
+SUDO_MODE_MINUTES_VALID = 5
 
-def verify_api_key(f):
-    @wraps(f)
-    def decorated(*args, **kwargs):
-        api_code = request.headers.get("Authentication")
-        api_key = ApiKey.get_by(code=api_code)
 
-        if not api_key:
+def authorize_request() -> Optional[Tuple[str, int]]:
+    api_code = request.headers.get("Authentication")
+    api_key = ApiKey.get_by(code=api_code)
+
+    if not api_key:
+        if current_user.is_authenticated:
+            g.user = current_user
+        else:
             return jsonify(error="Wrong api key"), 401
-
+    else:
         # Update api key stats
         api_key.last_used = arrow.now()
         api_key.times += 1
-        db.session.commit()
+        Session.commit()
 
         g.user = api_key.user
 
+    if g.user.disabled:
+        return jsonify(error="Disabled account"), 403
+
+    if not g.user.is_active():
+        return jsonify(error="Account does not exist"), 401
+
+    g.api_key = api_key
+    return None
+
+
+def check_sudo_mode_is_active(api_key: ApiKey) -> bool:
+    return api_key.sudo_mode_at and g.api_key.sudo_mode_at >= arrow.now().shift(
+        minutes=-SUDO_MODE_MINUTES_VALID
+    )
+
+
+def require_api_auth(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        error_return = authorize_request()
+        if error_return:
+            return error_return
+        return f(*args, **kwargs)
+
+    return decorated
+
+
+def require_api_sudo(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        error_return = authorize_request()
+        if error_return:
+            return error_return
+        if not check_sudo_mode_is_active(g.api_key):
+            return jsonify(error="Need sudo"), 440
         return f(*args, **kwargs)
 
     return decorated

app/api/serializer.py

@@ -0,0 +1,382 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from arrow import Arrow
+from sqlalchemy import or_, func, case, and_
+from sqlalchemy.orm import joinedload
+
+from app.config import PAGE_LIMIT
+from app.db import Session
+from app.models import (
+    Alias,
+    Contact,
+    EmailLog,
+    Mailbox,
+    AliasMailbox,
+    CustomDomain,
+    User,
+)
+
+
+@dataclass
+class AliasInfo:
+    alias: Alias
+    mailbox: Mailbox
+    mailboxes: [Mailbox]
+
+    nb_forward: int
+    nb_blocked: int
+    nb_reply: int
+
+    latest_email_log: EmailLog = None
+    latest_contact: Contact = None
+    custom_domain: Optional[CustomDomain] = None
+
+    def contain_mailbox(self, mailbox_id: int) -> bool:
+        return mailbox_id in [m.id for m in self.mailboxes]
+
+
+def serialize_alias_info(alias_info: AliasInfo) -> dict:
+    return {
+        # Alias field
+        "id": alias_info.alias.id,
+        "email": alias_info.alias.email,
+        "creation_date": alias_info.alias.created_at.format(),
+        "creation_timestamp": alias_info.alias.created_at.timestamp,
+        "enabled": alias_info.alias.enabled,
+        "note": alias_info.alias.note,
+        # activity
+        "nb_forward": alias_info.nb_forward,
+        "nb_block": alias_info.nb_blocked,
+        "nb_reply": alias_info.nb_reply,
+    }
+
+
+def serialize_alias_info_v2(alias_info: AliasInfo) -> dict:
+    res = {
+        # Alias field
+        "id": alias_info.alias.id,
+        "email": alias_info.alias.email,
+        "creation_date": alias_info.alias.created_at.format(),
+        "creation_timestamp": alias_info.alias.created_at.timestamp,
+        "enabled": alias_info.alias.enabled,
+        "note": alias_info.alias.note,
+        "name": alias_info.alias.name,
+        # activity
+        "nb_forward": alias_info.nb_forward,
+        "nb_block": alias_info.nb_blocked,
+        "nb_reply": alias_info.nb_reply,
+        # mailbox
+        "mailbox": {"id": alias_info.mailbox.id, "email": alias_info.mailbox.email},
+        "mailboxes": [
+            {"id": mailbox.id, "email": mailbox.email}
+            for mailbox in alias_info.mailboxes
+        ],
+        "support_pgp": alias_info.alias.mailbox_support_pgp(),
+        "disable_pgp": alias_info.alias.disable_pgp,
+        "latest_activity": None,
+        "pinned": alias_info.alias.pinned,
+    }
+    if alias_info.latest_email_log:
+        email_log = alias_info.latest_email_log
+        contact = alias_info.latest_contact
+        # latest activity
+        res["latest_activity"] = {
+            "timestamp": email_log.created_at.timestamp,
+            "action": email_log.get_action(),
+            "contact": {
+                "email": contact.website_email,
+                "name": contact.name,
+                "reverse_alias": contact.website_send_to(),
+            },
+        }
+    return res
+
+
+def serialize_contact(contact: Contact, existed=False) -> dict:
+    res = {
+        "id": contact.id,
+        "creation_date": contact.created_at.format(),
+        "creation_timestamp": contact.created_at.timestamp,
+        "last_email_sent_date": None,
+        "last_email_sent_timestamp": None,
+        "contact": contact.website_email,
+        "reverse_alias": contact.website_send_to(),
+        "reverse_alias_address": contact.reply_email,
+        "existed": existed,
+        "block_forward": contact.block_forward,
+    }
+
+    email_log: EmailLog = contact.last_reply()
+    if email_log:
+        res["last_email_sent_date"] = email_log.created_at.format()
+        res["last_email_sent_timestamp"] = email_log.created_at.timestamp
+
+    return res
+
+
+def get_alias_infos_with_pagination(user, page_id=0, query=None) -> [AliasInfo]:
+    ret = []
+    q = (
+        Session.query(Alias)
+        .options(joinedload(Alias.mailbox))
+        .filter(Alias.user_id == user.id)
+        .order_by(Alias.created_at.desc())
+    )
+
+    if query:
+        q = q.filter(
+            or_(Alias.email.ilike(f"%{query}%"), Alias.note.ilike(f"%{query}%"))
+        )
+
+    q = q.limit(PAGE_LIMIT).offset(page_id * PAGE_LIMIT)
+
+    for alias in q:
+        ret.append(get_alias_info(alias))
+
+    return ret
+
+
+def get_alias_infos_with_pagination_v3(
+    user,
+    page_id=0,
+    query=None,
+    sort=None,
+    alias_filter=None,
+    mailbox_id=None,
+    directory_id=None,
+    page_limit=PAGE_LIMIT,
+    page_size=PAGE_LIMIT,
+) -> [AliasInfo]:
+    q = construct_alias_query(user)
+
+    if query:
+        q = q.filter(
+            or_(
+                Alias.email.ilike(f"%{query}%"),
+                Alias.note.ilike(f"%{query}%"),
+                # can't use match() here as it uses to_tsquery that expected a tsquery input
+                # Alias.ts_vector.match(query),
+                Alias.ts_vector.op("@@")(func.plainto_tsquery("english", query)),
+                Alias.name.ilike(f"%{query}%"),
+            )
+        )
+
+    if mailbox_id:
+        q = q.join(
+            AliasMailbox, Alias.id == AliasMailbox.alias_id, isouter=True
+        ).filter(
+            or_(Alias.mailbox_id == mailbox_id, AliasMailbox.mailbox_id == mailbox_id)
+        )
+
+    if directory_id:
+        q = q.filter(Alias.directory_id == directory_id)
+
+    if alias_filter == "enabled":
+        q = q.filter(Alias.enabled)
+    elif alias_filter == "disabled":
+        q = q.filter(Alias.enabled.is_(False))
+    elif alias_filter == "pinned":
+        q = q.filter(Alias.pinned)
+    elif alias_filter == "hibp":
+        q = q.filter(Alias.hibp_breaches.any())
+
+    if sort == "old2new":
+        q = q.order_by(Alias.created_at)
+    elif sort == "new2old":
+        q = q.order_by(Alias.created_at.desc())
+    elif sort == "a2z":
+        q = q.order_by(Alias.email)
+    elif sort == "z2a":
+        q = q.order_by(Alias.email.desc())
+    else:
+        # default sorting
+        latest_activity = case(
+            [
+                (Alias.created_at > EmailLog.created_at, Alias.created_at),
+                (Alias.created_at < EmailLog.created_at, EmailLog.created_at),
+            ],
+            else_=Alias.created_at,
+        )
+        q = q.order_by(Alias.pinned.desc())
+        q = q.order_by(latest_activity.desc())
+
+    q = q.limit(page_limit).offset(page_id * page_size)
+
+    ret = []
+    for alias, contact, email_log, nb_reply, nb_blocked, nb_forward in list(q):
+        ret.append(
+            AliasInfo(
+                alias=alias,
+                mailbox=alias.mailbox,
+                mailboxes=alias.mailboxes,
+                nb_forward=nb_forward,
+                nb_blocked=nb_blocked,
+                nb_reply=nb_reply,
+                latest_email_log=email_log,
+                latest_contact=contact,
+                custom_domain=alias.custom_domain,
+            )
+        )
+
+    return ret
+
+
+def get_alias_info(alias: Alias) -> AliasInfo:
+    q = (
+        Session.query(Contact, EmailLog)
+        .filter(Contact.alias_id == alias.id)
+        .filter(EmailLog.contact_id == Contact.id)
+    )
+
+    alias_info = AliasInfo(
+        alias=alias,
+        nb_blocked=0,
+        nb_forward=0,
+        nb_reply=0,
+        mailbox=alias.mailbox,
+        mailboxes=[alias.mailbox],
+    )
+
+    for _, el in q:
+        if el.is_reply:
+            alias_info.nb_reply += 1
+        elif el.blocked:
+            alias_info.nb_blocked += 1
+        else:
+            alias_info.nb_forward += 1
+
+    return alias_info
+
+
+def get_alias_info_v2(alias: Alias, mailbox=None) -> AliasInfo:
+    if not mailbox:
+        mailbox = alias.mailbox
+
+    q = (
+        Session.query(Contact, EmailLog)
+        .filter(Contact.alias_id == alias.id)
+        .filter(EmailLog.contact_id == Contact.id)
+    )
+
+    latest_activity: Arrow = alias.created_at
+    latest_email_log = None
+    latest_contact = None
+
+    alias_info = AliasInfo(
+        alias=alias,
+        nb_blocked=0,
+        nb_forward=0,
+        nb_reply=0,
+        mailbox=mailbox,
+        mailboxes=[mailbox],
+    )
+
+    for m in alias._mailboxes:
+        alias_info.mailboxes.append(m)
+
+    # remove duplicates
+    # can happen that alias.mailbox_id also appears in AliasMailbox table
+    alias_info.mailboxes = list(set(alias_info.mailboxes))
+
+    for contact, email_log in q:
+        if email_log.is_reply:
+            alias_info.nb_reply += 1
+        elif email_log.blocked:
+            alias_info.nb_blocked += 1
+        else:
+            alias_info.nb_forward += 1
+
+        if email_log.created_at > latest_activity:
+            latest_activity = email_log.created_at
+            latest_email_log = email_log
+            latest_contact = contact
+
+    alias_info.latest_contact = latest_contact
+    alias_info.latest_email_log = latest_email_log
+
+    return alias_info
+
+
+def get_alias_contacts(alias, page_id: int) -> [dict]:
+    q = (
+        Contact.filter_by(alias_id=alias.id)
+        .order_by(Contact.id.desc())
+        .limit(PAGE_LIMIT)
+        .offset(page_id * PAGE_LIMIT)
+    )
+
+    res = []
+    for fe in q.all():
+        res.append(serialize_contact(fe))
+
+    return res
+
+
+def get_alias_info_v3(user: User, alias_id: int) -> AliasInfo:
+    # use the same query construction in get_alias_infos_with_pagination_v3
+    q = construct_alias_query(user)
+    q = q.filter(Alias.id == alias_id)
+
+    for alias, contact, email_log, nb_reply, nb_blocked, nb_forward in q:
+        return AliasInfo(
+            alias=alias,
+            mailbox=alias.mailbox,
+            mailboxes=alias.mailboxes,
+            nb_forward=nb_forward,
+            nb_blocked=nb_blocked,
+            nb_reply=nb_reply,
+            latest_email_log=email_log,
+            latest_contact=contact,
+            custom_domain=alias.custom_domain,
+        )
+
+
+def construct_alias_query(user: User):
+    # subquery on alias annotated with nb_reply, nb_blocked, nb_forward, max_created_at, latest_email_log_created_at
+    alias_activity_subquery = (
+        Session.query(
+            Alias.id,
+            func.sum(case([(EmailLog.is_reply, 1)], else_=0)).label("nb_reply"),
+            func.sum(
+                case(
+                    [(and_(EmailLog.is_reply.is_(False), EmailLog.blocked), 1)],
+                    else_=0,
+                )
+            ).label("nb_blocked"),
+            func.sum(
+                case(
+                    [
+                        (
+                            and_(
+                                EmailLog.is_reply.is_(False),
+                                EmailLog.blocked.is_(False),
+                            ),
+                            1,
+                        )
+                    ],
+                    else_=0,
+                )
+            ).label("nb_forward"),
+        )
+        .join(EmailLog, Alias.id == EmailLog.alias_id, isouter=True)
+        .filter(Alias.user_id == user.id)
+        .group_by(Alias.id)
+        .subquery()
+    )
+
+    return (
+        Session.query(
+            Alias,
+            Contact,
+            EmailLog,
+            alias_activity_subquery.c.nb_reply,
+            alias_activity_subquery.c.nb_blocked,
+            alias_activity_subquery.c.nb_forward,
+        )
+        .options(joinedload(Alias.hibp_breaches))
+        .options(joinedload(Alias.custom_domain))
+        .join(EmailLog, Alias.last_email_log_id == EmailLog.id, isouter=True)
+        .join(Contact, EmailLog.contact_id == Contact.id, isouter=True)
+        .filter(Alias.id == alias_activity_subquery.c.id)
+    )

app/api/views/alias.py

@@ -1,17 +1,37 @@
+from deprecated import deprecated
 from flask import g
-from flask import jsonify, request
-from flask_cors import cross_origin
+from flask import jsonify
+from flask import request
 
-from app.api.base import api_bp, verify_api_key
+from app import alias_utils
+from app.api.base import api_bp, require_api_auth
+from app.api.serializer import (
+    AliasInfo,
+    serialize_alias_info,
+    serialize_contact,
+    get_alias_infos_with_pagination,
+    get_alias_contacts,
+    serialize_alias_info_v2,
+    get_alias_info_v2,
+    get_alias_infos_with_pagination_v3,
+)
+from app.dashboard.views.alias_contact_manager import create_contact
 from app.dashboard.views.alias_log import get_alias_log
-from app.dashboard.views.index import get_alias_info, AliasInfo
-from app.extensions import db
-from app.models import GenEmail
+from app.db import Session
+from app.errors import (
+    CannotCreateContactForReverseAlias,
+    ErrContactErrorUpgradeNeeded,
+    ErrContactAlreadyExists,
+    ErrAddressInvalid,
+)
+from app.extensions import limiter
+from app.models import Alias, Contact, Mailbox, AliasMailbox
 
 
-@api_bp.route("/aliases")
-@cross_origin()
-@verify_api_key
+@deprecated
+@api_bp.route("/aliases", methods=["GET", "POST"])
+@require_api_auth
+@limiter.limit("10/minute", key_func=lambda: g.user.id)
 def get_aliases():
     """
     Get aliases
@@ -35,32 +55,96 @@ def get_aliases():
     except (ValueError, TypeError):
         return jsonify(error="page_id must be provided in request query"), 400
 
-    aliases: [AliasInfo] = get_alias_info(user, page_id=page_id)
+    query = None
+    data = request.get_json(silent=True)
+    if data:
+        query = data.get("query")
+
+    alias_infos: [AliasInfo] = get_alias_infos_with_pagination(
+        user, page_id=page_id, query=query
+    )
 
     return (
         jsonify(
-            aliases=[
-                {
-                    "id": alias.id,
-                    "email": alias.gen_email.email,
-                    "creation_date": alias.gen_email.created_at.format(),
-                    "creation_timestamp": alias.gen_email.created_at.timestamp,
-                    "nb_forward": alias.nb_forward,
-                    "nb_block": alias.nb_blocked,
-                    "nb_reply": alias.nb_reply,
-                    "enabled": alias.gen_email.enabled,
-                    "note": alias.note,
-                }
-                for alias in aliases
-            ]
+            aliases=[serialize_alias_info(alias_info) for alias_info in alias_infos]
+        ),
+        200,
+    )
+
+
+@api_bp.route("/v2/aliases", methods=["GET", "POST"])
+@require_api_auth
+@limiter.limit("50/minute", key_func=lambda: g.user.id)
+def get_aliases_v2():
+    """
+    Get aliases
+    Input:
+        page_id: in query
+        pinned: in query
+        disabled: in query
+        enabled: in query
+    Output:
+        - aliases: list of alias:
+            - id
+            - email
+            - creation_date
+            - creation_timestamp
+            - nb_forward
+            - nb_block
+            - nb_reply
+            - note
+            - mailbox
+            - mailboxes
+            - support_pgp
+            - disable_pgp
+            - latest_activity: null if no activity.
+                - timestamp
+                - action: forward|reply|block|bounced
+                - contact:
+                    - email
+                    - name
+                    - reverse_alias
+
+
+    """
+    user = g.user
+    try:
+        page_id = int(request.args.get("page_id"))
+    except (ValueError, TypeError):
+        return jsonify(error="page_id must be provided in request query"), 400
+
+    pinned = "pinned" in request.args
+    disabled = "disabled" in request.args
+    enabled = "enabled" in request.args
+
+    if pinned:
+        alias_filter = "pinned"
+    elif disabled:
+        alias_filter = "disabled"
+    elif enabled:
+        alias_filter = "enabled"
+    else:
+        alias_filter = None
+
+    query = None
+    data = request.get_json(silent=True)
+    if data:
+        query = data.get("query")
+
+    alias_infos: [AliasInfo] = get_alias_infos_with_pagination_v3(
+        user, page_id=page_id, query=query, alias_filter=alias_filter
+    )
+
+    return (
+        jsonify(
+            aliases=[serialize_alias_info_v2(alias_info) for alias_info in alias_infos]
         ),
         200,
     )
 
 
 @api_bp.route("/aliases/<int:alias_id>", methods=["DELETE"])
-@cross_origin()
-@verify_api_key
+@require_api_auth
 def delete_alias(alias_id):
     """
     Delete alias
@@ -71,20 +155,18 @@ def delete_alias(alias_id):
 
     """
     user = g.user
-    gen_email = GenEmail.get(alias_id)
+    alias = Alias.get(alias_id)
 
-    if gen_email.user_id != user.id:
+    if not alias or alias.user_id != user.id:
         return jsonify(error="Forbidden"), 403
 
-    GenEmail.delete(alias_id)
-    db.session.commit()
+    alias_utils.delete_alias(alias, user)
 
     return jsonify(deleted=True), 200
 
 
 @api_bp.route("/aliases/<int:alias_id>/toggle", methods=["POST"])
-@cross_origin()
-@verify_api_key
+@require_api_auth
 def toggle_alias(alias_id):
     """
     Enable/disable alias
@@ -97,20 +179,19 @@ def toggle_alias(alias_id):
 
     """
     user = g.user
-    gen_email: GenEmail = GenEmail.get(alias_id)
+    alias: Alias = Alias.get(alias_id)
 
-    if gen_email.user_id != user.id:
+    if not alias or alias.user_id != user.id:
         return jsonify(error="Forbidden"), 403
 
-    gen_email.enabled = not gen_email.enabled
-    db.session.commit()
+    alias_utils.change_alias_status(alias, enabled=not alias.enabled)
+    Session.commit()
 
-    return jsonify(enabled=gen_email.enabled), 200
+    return jsonify(enabled=alias.enabled), 200
 
 
 @api_bp.route("/aliases/<int:alias_id>/activities")
-@cross_origin()
-@verify_api_key
+@require_api_auth
 def get_alias_activities(alias_id):
     """
     Get aliases
@@ -121,7 +202,8 @@ def get_alias_activities(alias_id):
             - from
             - to
             - timestamp
-            - action: forward|reply|block
+            - action: forward|reply|block|bounced
+            - reverse_alias
 
     """
     user = g.user
@@ -130,23 +212,27 @@ def get_alias_activities(alias_id):
     except (ValueError, TypeError):
         return jsonify(error="page_id must be provided in request query"), 400
 
-    gen_email: GenEmail = GenEmail.get(alias_id)
+    alias: Alias = Alias.get(alias_id)
 
-    if gen_email.user_id != user.id:
+    if not alias or alias.user_id != user.id:
         return jsonify(error="Forbidden"), 403
 
-    alias_logs = get_alias_log(gen_email, page_id)
+    alias_logs = get_alias_log(alias, page_id)
 
     activities = []
     for alias_log in alias_logs:
-        activity = {"timestamp": alias_log.when.timestamp}
+        activity = {
+            "timestamp": alias_log.when.timestamp,
+            "reverse_alias": alias_log.reverse_alias,
+            "reverse_alias_address": alias_log.contact.reply_email,
+        }
         if alias_log.is_reply:
             activity["from"] = alias_log.alias
-            activity["to"] = alias_log.website_from or alias_log.website_email
+            activity["to"] = alias_log.website_email
             activity["action"] = "reply"
         else:
             activity["to"] = alias_log.alias
-            activity["from"] = alias_log.website_from or alias_log.website_email
+            activity["from"] = alias_log.website_email
 
             if alias_log.bounced:
                 activity["action"] = "bounced"
@@ -157,4 +243,235 @@ def get_alias_activities(alias_id):
 
         activities.append(activity)
 
-    return (jsonify(activities=activities), 200)
+    return jsonify(activities=activities), 200
+
+
+@api_bp.route("/aliases/<int:alias_id>", methods=["PUT", "PATCH"])
+@require_api_auth
+def update_alias(alias_id):
+    """
+    Update alias note
+    Input:
+        alias_id: in url
+        note (optional): in body
+        name (optional): in body
+        mailbox_id (optional): in body
+        disable_pgp (optional): in body
+    Output:
+        200
+    """
+    data = request.get_json()
+    if not data:
+        return jsonify(error="request body cannot be empty"), 400
+
+    user = g.user
+    alias: Alias = Alias.get(alias_id)
+
+    if not alias or alias.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    changed = False
+    if "note" in data:
+        new_note = data.get("note")
+        alias.note = new_note
+        changed = True
+
+    if "mailbox_id" in data:
+        mailbox_id = int(data.get("mailbox_id"))
+        mailbox = Mailbox.get(mailbox_id)
+        if not mailbox or mailbox.user_id != user.id or not mailbox.verified:
+            return jsonify(error="Forbidden"), 400
+
+        alias.mailbox_id = mailbox_id
+        changed = True
+
+    if "mailbox_ids" in data:
+        mailbox_ids = [int(m_id) for m_id in data.get("mailbox_ids")]
+        mailboxes: [Mailbox] = []
+
+        # check if all mailboxes belong to user
+        for mailbox_id in mailbox_ids:
+            mailbox = Mailbox.get(mailbox_id)
+            if not mailbox or mailbox.user_id != user.id or not mailbox.verified:
+                return jsonify(error="Forbidden"), 400
+            mailboxes.append(mailbox)
+
+        if not mailboxes:
+            return jsonify(error="Must choose at least one mailbox"), 400
+
+        # <<< update alias mailboxes >>>
+        # first remove all existing alias-mailboxes links
+        AliasMailbox.filter_by(alias_id=alias.id).delete()
+        Session.flush()
+
+        # then add all new mailboxes
+        for i, mailbox in enumerate(mailboxes):
+            if i == 0:
+                alias.mailbox_id = mailboxes[0].id
+            else:
+                AliasMailbox.create(alias_id=alias.id, mailbox_id=mailbox.id)
+        # <<< END update alias mailboxes >>>
+
+        changed = True
+
+    if "name" in data:
+        # to make sure alias name doesn't contain linebreak
+        new_name = data.get("name")
+        if new_name and len(new_name) > 128:
+            return jsonify(error="Name can't be longer than 128 characters"), 400
+
+        if new_name:
+            new_name = new_name.replace("\n", "")
+        alias.name = new_name
+        changed = True
+
+    if "disable_pgp" in data:
+        alias.disable_pgp = data.get("disable_pgp")
+        changed = True
+
+    if "pinned" in data:
+        alias.pinned = data.get("pinned")
+        changed = True
+
+    if changed:
+        Session.commit()
+
+    return jsonify(ok=True), 200
+
+
+@api_bp.route("/aliases/<int:alias_id>", methods=["GET"])
+@require_api_auth
+def get_alias(alias_id):
+    """
+    Get alias
+    Input:
+        alias_id: in url
+    Output:
+        Alias info, same as in get_aliases
+
+    """
+    user = g.user
+    alias: Alias = Alias.get(alias_id)
+
+    if not alias:
+        return jsonify(error="Unknown error"), 400
+
+    if alias.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    return jsonify(**serialize_alias_info_v2(get_alias_info_v2(alias))), 200
+
+
+@api_bp.route("/aliases/<int:alias_id>/contacts")
+@require_api_auth
+def get_alias_contacts_route(alias_id):
+    """
+    Get alias contacts
+    Input:
+        page_id: in query
+    Output:
+        - contacts: list of contacts:
+            - creation_date
+            - creation_timestamp
+            - last_email_sent_date
+            - last_email_sent_timestamp
+            - contact
+            - reverse_alias
+
+    """
+    user = g.user
+    try:
+        page_id = int(request.args.get("page_id"))
+    except (ValueError, TypeError):
+        return jsonify(error="page_id must be provided in request query"), 400
+
+    alias: Alias = Alias.get(alias_id)
+
+    if not alias:
+        return jsonify(error="No such alias"), 404
+
+    if alias.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    contacts = get_alias_contacts(alias, page_id)
+
+    return jsonify(contacts=contacts), 200
+
+
+@api_bp.route("/aliases/<int:alias_id>/contacts", methods=["POST"])
+@require_api_auth
+def create_contact_route(alias_id):
+    """
+    Create contact for an alias
+    Input:
+        alias_id: in url
+        contact: in body
+    Output:
+        201 if success
+        409 if contact already added
+    """
+    data = request.get_json()
+    if not data:
+        return jsonify(error="request body cannot be empty"), 400
+
+    alias: Alias = Alias.get(alias_id)
+
+    if alias.user_id != g.user.id:
+        return jsonify(error="Forbidden"), 403
+
+    contact_address = data.get("contact")
+
+    try:
+        contact = create_contact(g.user, alias, contact_address)
+    except ErrContactErrorUpgradeNeeded as err:
+        return jsonify(error=err.error_for_user()), 403
+    except (ErrAddressInvalid, CannotCreateContactForReverseAlias) as err:
+        return jsonify(error=err.error_for_user()), 400
+    except ErrContactAlreadyExists as err:
+        return jsonify(**serialize_contact(err.contact, existed=True)), 200
+
+    return jsonify(**serialize_contact(contact)), 201
+
+
+@api_bp.route("/contacts/<int:contact_id>", methods=["DELETE"])
+@require_api_auth
+def delete_contact(contact_id):
+    """
+    Delete contact
+    Input:
+        contact_id: in url
+    Output:
+        200
+    """
+    user = g.user
+    contact = Contact.get(contact_id)
+
+    if not contact or contact.alias.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    Contact.delete(contact_id)
+    Session.commit()
+
+    return jsonify(deleted=True), 200
+
+
+@api_bp.route("/contacts/<int:contact_id>/toggle", methods=["POST"])
+@require_api_auth
+def toggle_contact(contact_id):
+    """
+    Block/Unblock contact
+    Input:
+        contact_id: in url
+    Output:
+        200
+    """
+    user = g.user
+    contact = Contact.get(contact_id)
+
+    if not contact or contact.alias.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    contact.block_forward = not contact.block_forward
+    Session.commit()
+
+    return jsonify(block_forward=contact.block_forward), 200

app/api/views/alias_options.py

@@ -1,105 +1,28 @@
+import tldextract
 from flask import jsonify, request, g
-from flask_cors import cross_origin
 from sqlalchemy import desc
 
-from app.api.base import api_bp, verify_api_key
-from app.config import ALIAS_DOMAINS, DISABLE_ALIAS_SUFFIX
-from app.extensions import db
+from app.alias_suffix import get_alias_suffixes
+from app.api.base import api_bp, require_api_auth
+from app.db import Session
 from app.log import LOG
-from app.models import AliasUsedOn, GenEmail, User
-from app.utils import convert_to_id, random_word
+from app.models import AliasUsedOn, Alias, User
+from app.utils import convert_to_id
 
 
-@api_bp.route("/alias/options")
-@cross_origin()
-@verify_api_key
-def options():
-    """
-    Return what options user has when creating new alias.
-    Input:
-        a valid api-key in "Authentication" header and
-        optional "hostname" in args
-    Output: cf README
-        optional recommendation:
-        optional custom
-        can_create_custom: boolean
-        existing: array of existing aliases
-
-    """
-    LOG.error("/v2/alias/options should be used instead")
-    user = g.user
-    hostname = request.args.get("hostname")
-
-    ret = {
-        "existing": [ge.email for ge in GenEmail.query.filter_by(user_id=user.id)],
-        "can_create_custom": user.can_create_new_alias(),
-    }
-
-    # recommendation alias if exist
-    if hostname:
-        # put the latest used alias first
-        q = (
-            db.session.query(AliasUsedOn, GenEmail, User)
-            .filter(
-                AliasUsedOn.gen_email_id == GenEmail.id,
-                GenEmail.user_id == user.id,
-                AliasUsedOn.hostname == hostname,
-            )
-            .order_by(desc(AliasUsedOn.created_at))
-        )
-
-        r = q.first()
-        if r:
-            _, alias, _ = r
-            LOG.d("found alias %s %s %s", alias, hostname, user)
-            ret["recommendation"] = {"alias": alias.email, "hostname": hostname}
-
-    # custom alias suggestion and suffix
-    ret["custom"] = {}
-    if hostname:
-        # keep only the domain name of hostname, ignore TLD and subdomain
-        # for ex www.groupon.com -> groupon
-        domain_name = hostname
-        if "." in hostname:
-            parts = hostname.split(".")
-            domain_name = parts[-2]
-            domain_name = convert_to_id(domain_name)
-        ret["custom"]["suggestion"] = domain_name
-    else:
-        ret["custom"]["suggestion"] = ""
-
-    ret["custom"]["suffixes"] = []
-    # maybe better to make sure the suffix is never used before
-    # but this is ok as there's a check when creating a new custom alias
-    for domain in ALIAS_DOMAINS:
-        if DISABLE_ALIAS_SUFFIX:
-            ret["custom"]["suffixes"].append(f"@{domain}")
-        else:
-            ret["custom"]["suffixes"].append(f".{random_word()}@{domain}")
-
-    for custom_domain in user.verified_custom_domains():
-        ret["custom"]["suffixes"].append("@" + custom_domain.domain)
-
-    # custom domain should be put first
-    ret["custom"]["suffixes"] = list(reversed(ret["custom"]["suffixes"]))
-
-    return jsonify(ret)
-
-
-@api_bp.route("/v2/alias/options")
-@cross_origin()
-@verify_api_key
-def options_v2():
+@api_bp.route("/v4/alias/options")
+@require_api_auth
+def options_v4():
     """
     Return what options user has when creating new alias.
+    Same as v3 but return time-based signed-suffix in addition to suffix. To be used with /v2/alias/custom/new
     Input:
         a valid api-key in "Authentication" header and
         optional "hostname" in args
     Output: cf README
         can_create: bool
-        suffixes: [str]
+        suffixes: [[suffix, signed_suffix]]
         prefix_suggestion: str
-        existing: [str]
         recommendation: Optional dict
             alias: str
             hostname: str
@@ -110,9 +33,6 @@ def options_v2():
     hostname = request.args.get("hostname")
 
     ret = {
-        "existing": [
-            ge.email for ge in GenEmail.query.filter_by(user_id=user.id, enabled=True)
-        ],
         "can_create": user.can_create_new_alias(),
         "suffixes": [],
         "prefix_suggestion": "",
@@ -122,10 +42,10 @@ def options_v2():
     if hostname:
         # put the latest used alias first
         q = (
-            db.session.query(AliasUsedOn, GenEmail, User)
+            Session.query(AliasUsedOn, Alias, User)
             .filter(
-                AliasUsedOn.gen_email_id == GenEmail.id,
-                GenEmail.user_id == user.id,
+                AliasUsedOn.alias_id == Alias.id,
+                Alias.user_id == user.id,
                 AliasUsedOn.hostname == hostname,
             )
             .order_by(desc(AliasUsedOn.created_at))
@@ -141,25 +61,93 @@ def options_v2():
     if hostname:
         # keep only the domain name of hostname, ignore TLD and subdomain
         # for ex www.groupon.com -> groupon
-        domain_name = hostname
-        if "." in hostname:
-            parts = hostname.split(".")
-            domain_name = parts[-2]
-            domain_name = convert_to_id(domain_name)
-        ret["prefix_suggestion"] = domain_name
+        ext = tldextract.extract(hostname)
+        prefix_suggestion = ext.domain
+        prefix_suggestion = convert_to_id(prefix_suggestion)
+        ret["prefix_suggestion"] = prefix_suggestion
 
-    # maybe better to make sure the suffix is never used before
-    # but this is ok as there's a check when creating a new custom alias
-    for domain in ALIAS_DOMAINS:
-        if DISABLE_ALIAS_SUFFIX:
-            ret["suffixes"].append(f"@{domain}")
-        else:
-            ret["suffixes"].append(f".{random_word()}@{domain}")
-
-    for custom_domain in user.verified_custom_domains():
-        ret["suffixes"].append("@" + custom_domain.domain)
+    suffixes = get_alias_suffixes(user)
 
     # custom domain should be put first
-    ret["suffixes"] = list(reversed(ret["suffixes"]))
+    ret["suffixes"] = list([suffix.suffix, suffix.signed_suffix] for suffix in suffixes)
+
+    return jsonify(ret)
+
+
+@api_bp.route("/v5/alias/options")
+@require_api_auth
+def options_v5():
+    """
+    Return what options user has when creating new alias.
+    Same as v4 but uses a better format. To be used with /v2/alias/custom/new
+    Input:
+        a valid api-key in "Authentication" header and
+        optional "hostname" in args
+    Output: cf README
+        can_create: bool
+        suffixes: [
+            {
+                suffix: "suffix",
+                signed_suffix: "signed_suffix",
+                is_custom: true,
+                is_premium: false
+            }
+        ]
+        prefix_suggestion: str
+        recommendation: Optional dict
+            alias: str
+            hostname: str
+
+
+    """
+    user = g.user
+    hostname = request.args.get("hostname")
+
+    ret = {
+        "can_create": user.can_create_new_alias(),
+        "suffixes": [],
+        "prefix_suggestion": "",
+    }
+
+    # recommendation alias if exist
+    if hostname:
+        # put the latest used alias first
+        q = (
+            Session.query(AliasUsedOn, Alias, User)
+            .filter(
+                AliasUsedOn.alias_id == Alias.id,
+                Alias.user_id == user.id,
+                AliasUsedOn.hostname == hostname,
+            )
+            .order_by(desc(AliasUsedOn.created_at))
+        )
+
+        r = q.first()
+        if r:
+            _, alias, _ = r
+            LOG.d("found alias %s %s %s", alias, hostname, user)
+            ret["recommendation"] = {"alias": alias.email, "hostname": hostname}
+
+    # custom alias suggestion and suffix
+    if hostname:
+        # keep only the domain name of hostname, ignore TLD and subdomain
+        # for ex www.groupon.com -> groupon
+        ext = tldextract.extract(hostname)
+        prefix_suggestion = ext.domain
+        prefix_suggestion = convert_to_id(prefix_suggestion)
+        ret["prefix_suggestion"] = prefix_suggestion
+
+    suffixes = get_alias_suffixes(user)
+
+    # custom domain should be put first
+    ret["suffixes"] = [
+        {
+            "suffix": suffix.suffix,
+            "signed_suffix": suffix.signed_suffix,
+            "is_custom": suffix.is_custom,
+            "is_premium": suffix.is_premium,
+        }
+        for suffix in suffixes
+    ]
 
     return jsonify(ret)

app/api/views/apple.py

@@ -0,0 +1,576 @@
+from typing import Optional
+
+import arrow
+import requests
+from flask import g
+from flask import jsonify
+from flask import request
+from requests import RequestException
+
+from app.api.base import api_bp, require_api_auth
+from app.config import APPLE_API_SECRET, MACAPP_APPLE_API_SECRET
+from app.subscription_webhook import execute_subscription_webhook
+from app.db import Session
+from app.log import LOG
+from app.models import PlanEnum, AppleSubscription
+
+_MONTHLY_PRODUCT_ID = "io.simplelogin.ios_app.subscription.premium.monthly"
+_YEARLY_PRODUCT_ID = "io.simplelogin.ios_app.subscription.premium.yearly"
+
+# SL Mac app used to be in SL account
+_MACAPP_MONTHLY_PRODUCT_ID = "io.simplelogin.macapp.subscription.premium.monthly"
+_MACAPP_YEARLY_PRODUCT_ID = "io.simplelogin.macapp.subscription.premium.yearly"
+
+# SL Mac app is moved to Proton account
+_MACAPP_MONTHLY_PRODUCT_ID_NEW = "me.proton.simplelogin.macos.premium.monthly"
+_MACAPP_YEARLY_PRODUCT_ID_NEW = "me.proton.simplelogin.macos.premium.yearly"
+
+# Apple API URL
+_SANDBOX_URL = "https://sandbox.itunes.apple.com/verifyReceipt"
+_PROD_URL = "https://buy.itunes.apple.com/verifyReceipt"
+
+
+@api_bp.route("/apple/process_payment", methods=["POST"])
+@require_api_auth
+def apple_process_payment():
+    """
+    Process payment
+    Input:
+        receipt_data: in body
+        (optional) is_macapp: in body
+    Output:
+        200 of the payment is successful, i.e. user is upgraded to premium
+
+    """
+    user = g.user
+    LOG.d("request for /apple/process_payment from %s", user)
+    data = request.get_json()
+    receipt_data = data.get("receipt_data")
+    is_macapp = "is_macapp" in data and data["is_macapp"] is True
+
+    if is_macapp:
+        LOG.d("Use Macapp secret")
+        password = MACAPP_APPLE_API_SECRET
+    else:
+        password = APPLE_API_SECRET
+
+    apple_sub = verify_receipt(receipt_data, user, password)
+    if apple_sub:
+        execute_subscription_webhook(user)
+        return jsonify(ok=True), 200
+
+    return jsonify(error="Processing failed"), 400
+
+
+@api_bp.route("/apple/update_notification", methods=["GET", "POST"])
+def apple_update_notification():
+    """
+    The "Subscription Status URL" to receive update notifications from Apple
+    """
+    # request.json looks like this
+    # will use unified_receipt.latest_receipt_info and NOT latest_expired_receipt_info
+    # more info on https://developer.apple.com/documentation/appstoreservernotifications/responsebody
+    # {
+    #     "unified_receipt": {
+    #         "latest_receipt": "long string",
+    #         "pending_renewal_info": [
+    #             {
+    #                 "is_in_billing_retry_period": "0",
+    #                 "auto_renew_status": "0",
+    #                 "original_transaction_id": "1000000654277043",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #                 "expiration_intent": "1",
+    #                 "auto_renew_product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #             }
+    #         ],
+    #         "environment": "Sandbox",
+    #         "status": 0,
+    #         "latest_receipt_info": [
+    #             {
+    #                 "expires_date_pst": "2020-04-20 21:11:57 America/Los_Angeles",
+    #                 "purchase_date": "2020-04-21 03:11:57 Etc/GMT",
+    #                 "purchase_date_ms": "1587438717000",
+    #                 "original_purchase_date_ms": "1587420715000",
+    #                 "transaction_id": "1000000654329911",
+    #                 "original_transaction_id": "1000000654277043",
+    #                 "quantity": "1",
+    #                 "expires_date_ms": "1587442317000",
+    #                 "original_purchase_date_pst": "2020-04-20 15:11:55 America/Los_Angeles",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #                 "subscription_group_identifier": "20624274",
+    #                 "web_order_line_item_id": "1000000051891577",
+    #                 "expires_date": "2020-04-21 04:11:57 Etc/GMT",
+    #                 "is_in_intro_offer_period": "false",
+    #                 "original_purchase_date": "2020-04-20 22:11:55 Etc/GMT",
+    #                 "purchase_date_pst": "2020-04-20 20:11:57 America/Los_Angeles",
+    #                 "is_trial_period": "false",
+    #             },
+    #             {
+    #                 "expires_date_pst": "2020-04-20 20:11:57 America/Los_Angeles",
+    #                 "purchase_date": "2020-04-21 02:11:57 Etc/GMT",
+    #                 "purchase_date_ms": "1587435117000",
+    #                 "original_purchase_date_ms": "1587420715000",
+    #                 "transaction_id": "1000000654313889",
+    #                 "original_transaction_id": "1000000654277043",
+    #                 "quantity": "1",
+    #                 "expires_date_ms": "1587438717000",
+    #                 "original_purchase_date_pst": "2020-04-20 15:11:55 America/Los_Angeles",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #                 "subscription_group_identifier": "20624274",
+    #                 "web_order_line_item_id": "1000000051890729",
+    #                 "expires_date": "2020-04-21 03:11:57 Etc/GMT",
+    #                 "is_in_intro_offer_period": "false",
+    #                 "original_purchase_date": "2020-04-20 22:11:55 Etc/GMT",
+    #                 "purchase_date_pst": "2020-04-20 19:11:57 America/Los_Angeles",
+    #                 "is_trial_period": "false",
+    #             },
+    #             {
+    #                 "expires_date_pst": "2020-04-20 19:11:54 America/Los_Angeles",
+    #                 "purchase_date": "2020-04-21 01:11:54 Etc/GMT",
+    #                 "purchase_date_ms": "1587431514000",
+    #                 "original_purchase_date_ms": "1587420715000",
+    #                 "transaction_id": "1000000654300800",
+    #                 "original_transaction_id": "1000000654277043",
+    #                 "quantity": "1",
+    #                 "expires_date_ms": "1587435114000",
+    #                 "original_purchase_date_pst": "2020-04-20 15:11:55 America/Los_Angeles",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #                 "subscription_group_identifier": "20624274",
+    #                 "web_order_line_item_id": "1000000051890161",
+    #                 "expires_date": "2020-04-21 02:11:54 Etc/GMT",
+    #                 "is_in_intro_offer_period": "false",
+    #                 "original_purchase_date": "2020-04-20 22:11:55 Etc/GMT",
+    #                 "purchase_date_pst": "2020-04-20 18:11:54 America/Los_Angeles",
+    #                 "is_trial_period": "false",
+    #             },
+    #             {
+    #                 "expires_date_pst": "2020-04-20 18:11:54 America/Los_Angeles",
+    #                 "purchase_date": "2020-04-21 00:11:54 Etc/GMT",
+    #                 "purchase_date_ms": "1587427914000",
+    #                 "original_purchase_date_ms": "1587420715000",
+    #                 "transaction_id": "1000000654293615",
+    #                 "original_transaction_id": "1000000654277043",
+    #                 "quantity": "1",
+    #                 "expires_date_ms": "1587431514000",
+    #                 "original_purchase_date_pst": "2020-04-20 15:11:55 America/Los_Angeles",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #                 "subscription_group_identifier": "20624274",
+    #                 "web_order_line_item_id": "1000000051889539",
+    #                 "expires_date": "2020-04-21 01:11:54 Etc/GMT",
+    #                 "is_in_intro_offer_period": "false",
+    #                 "original_purchase_date": "2020-04-20 22:11:55 Etc/GMT",
+    #                 "purchase_date_pst": "2020-04-20 17:11:54 America/Los_Angeles",
+    #                 "is_trial_period": "false",
+    #             },
+    #             {
+    #                 "expires_date_pst": "2020-04-20 17:11:54 America/Los_Angeles",
+    #                 "purchase_date": "2020-04-20 23:11:54 Etc/GMT",
+    #                 "purchase_date_ms": "1587424314000",
+    #                 "original_purchase_date_ms": "1587420715000",
+    #                 "transaction_id": "1000000654285464",
+    #                 "original_transaction_id": "1000000654277043",
+    #                 "quantity": "1",
+    #                 "expires_date_ms": "1587427914000",
+    #                 "original_purchase_date_pst": "2020-04-20 15:11:55 America/Los_Angeles",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #                 "subscription_group_identifier": "20624274",
+    #                 "web_order_line_item_id": "1000000051888827",
+    #                 "expires_date": "2020-04-21 00:11:54 Etc/GMT",
+    #                 "is_in_intro_offer_period": "false",
+    #                 "original_purchase_date": "2020-04-20 22:11:55 Etc/GMT",
+    #                 "purchase_date_pst": "2020-04-20 16:11:54 America/Los_Angeles",
+    #                 "is_trial_period": "false",
+    #             },
+    #             {
+    #                 "expires_date_pst": "2020-04-20 16:11:54 America/Los_Angeles",
+    #                 "purchase_date": "2020-04-20 22:11:54 Etc/GMT",
+    #                 "purchase_date_ms": "1587420714000",
+    #                 "original_purchase_date_ms": "1587420715000",
+    #                 "transaction_id": "1000000654277043",
+    #                 "original_transaction_id": "1000000654277043",
+    #                 "quantity": "1",
+    #                 "expires_date_ms": "1587424314000",
+    #                 "original_purchase_date_pst": "2020-04-20 15:11:55 America/Los_Angeles",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #                 "subscription_group_identifier": "20624274",
+    #                 "web_order_line_item_id": "1000000051888825",
+    #                 "expires_date": "2020-04-20 23:11:54 Etc/GMT",
+    #                 "is_in_intro_offer_period": "false",
+    #                 "original_purchase_date": "2020-04-20 22:11:55 Etc/GMT",
+    #                 "purchase_date_pst": "2020-04-20 15:11:54 America/Los_Angeles",
+    #                 "is_trial_period": "false",
+    #             },
+    #         ],
+    #     },
+    #     "auto_renew_status_change_date": "2020-04-21 04:11:33 Etc/GMT",
+    #     "environment": "Sandbox",
+    #     "auto_renew_status": "false",
+    #     "auto_renew_status_change_date_pst": "2020-04-20 21:11:33 America/Los_Angeles",
+    #     "latest_expired_receipt": "long string",
+    #     "latest_expired_receipt_info": {
+    #         "original_purchase_date_pst": "2020-04-20 15:11:55 America/Los_Angeles",
+    #         "quantity": "1",
+    #         "subscription_group_identifier": "20624274",
+    #         "unique_vendor_identifier": "4C4DF6BA-DE2A-4737-9A68-5992338886DC",
+    #         "original_purchase_date_ms": "1587420715000",
+    #         "expires_date_formatted": "2020-04-21 04:11:57 Etc/GMT",
+    #         "is_in_intro_offer_period": "false",
+    #         "purchase_date_ms": "1587438717000",
+    #         "expires_date_formatted_pst": "2020-04-20 21:11:57 America/Los_Angeles",
+    #         "is_trial_period": "false",
+    #         "item_id": "1508744966",
+    #         "unique_identifier": "b55fc3dcc688e979115af0697a0195be78be7cbd",
+    #         "original_transaction_id": "1000000654277043",
+    #         "expires_date": "1587442317000",
+    #         "transaction_id": "1000000654329911",
+    #         "bvrs": "3",
+    #         "web_order_line_item_id": "1000000051891577",
+    #         "version_external_identifier": "834289833",
+    #         "bid": "io.simplelogin.ios-app",
+    #         "product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #         "purchase_date": "2020-04-21 03:11:57 Etc/GMT",
+    #         "purchase_date_pst": "2020-04-20 20:11:57 America/Los_Angeles",
+    #         "original_purchase_date": "2020-04-20 22:11:55 Etc/GMT",
+    #     },
+    #     "password": "22b9d5a110dd4344a1681631f1f95f55",
+    #     "auto_renew_status_change_date_ms": "1587442293000",
+    #     "auto_renew_product_id": "io.simplelogin.ios_app.subscription.premium.yearly",
+    #     "notification_type": "DID_CHANGE_RENEWAL_STATUS",
+    # }
+    LOG.d("request for /api/apple/update_notification")
+    data = request.get_json()
+    if not (
+        data
+        and data.get("unified_receipt")
+        and data["unified_receipt"].get("latest_receipt_info")
+    ):
+        LOG.d("Invalid data %s", data)
+        return jsonify(error="Empty Response"), 400
+
+    transactions = data["unified_receipt"]["latest_receipt_info"]
+
+    # dict of original_transaction_id and transaction
+    latest_transactions = {}
+
+    for transaction in transactions:
+        original_transaction_id = transaction["original_transaction_id"]
+        if not latest_transactions.get(original_transaction_id):
+            latest_transactions[original_transaction_id] = transaction
+
+        if (
+            transaction["expires_date_ms"]
+            > latest_transactions[original_transaction_id]["expires_date_ms"]
+        ):
+            latest_transactions[original_transaction_id] = transaction
+
+    for original_transaction_id, transaction in latest_transactions.items():
+        expires_date = arrow.get(int(transaction["expires_date_ms"]) / 1000)
+        plan = (
+            PlanEnum.monthly
+            if transaction["product_id"]
+            in (
+                _MONTHLY_PRODUCT_ID,
+                _MACAPP_MONTHLY_PRODUCT_ID,
+                _MACAPP_MONTHLY_PRODUCT_ID_NEW,
+            )
+            else PlanEnum.yearly
+        )
+
+        apple_sub: AppleSubscription = AppleSubscription.get_by(
+            original_transaction_id=original_transaction_id
+        )
+
+        if apple_sub:
+            user = apple_sub.user
+            LOG.d(
+                "Update AppleSubscription for user %s, expired at %s, plan %s",
+                user,
+                expires_date,
+                plan,
+            )
+            apple_sub.receipt_data = data["unified_receipt"]["latest_receipt"]
+            apple_sub.expires_date = expires_date
+            apple_sub.plan = plan
+            apple_sub.product_id = transaction["product_id"]
+            Session.commit()
+            execute_subscription_webhook(user)
+            return jsonify(ok=True), 200
+        else:
+            LOG.w(
+                "No existing AppleSub for original_transaction_id %s",
+                original_transaction_id,
+            )
+            LOG.d("request data %s", data)
+            return jsonify(error="Processing failed"), 400
+
+
+def verify_receipt(receipt_data, user, password) -> Optional[AppleSubscription]:
+    """
+    Call https://buy.itunes.apple.com/verifyReceipt and create/update AppleSubscription table
+    Call the production URL for verifyReceipt first,
+        use sandbox URL if receive a 21007 status code.
+
+    Return AppleSubscription object if success
+
+    https://developer.apple.com/documentation/appstorereceipts/verifyreceipt
+    """
+    LOG.d("start verify_receipt")
+    try:
+        r = requests.post(
+            _PROD_URL, json={"receipt-data": receipt_data, "password": password}
+        )
+    except RequestException:
+        LOG.w("cannot call Apple server %s", _PROD_URL)
+        return None
+
+    if r.status_code >= 500:
+        LOG.w("Apple server error, response:%s %s", r, r.content)
+        return None
+
+    if r.json() == {"status": 21007}:
+        # try sandbox_url
+        LOG.w("Use the sandbox url instead")
+        r = requests.post(
+            _SANDBOX_URL,
+            json={"receipt-data": receipt_data, "password": password},
+        )
+
+    data = r.json()
+    # data has the following format
+    # {
+    #     "status": 0,
+    #     "environment": "Sandbox",
+    #     "receipt": {
+    #         "receipt_type": "ProductionSandbox",
+    #         "adam_id": 0,
+    #         "app_item_id": 0,
+    #         "bundle_id": "io.simplelogin.ios-app",
+    #         "application_version": "2",
+    #         "download_id": 0,
+    #         "version_external_identifier": 0,
+    #         "receipt_creation_date": "2020-04-18 16:36:34 Etc/GMT",
+    #         "receipt_creation_date_ms": "1587227794000",
+    #         "receipt_creation_date_pst": "2020-04-18 09:36:34 America/Los_Angeles",
+    #         "request_date": "2020-04-18 16:46:36 Etc/GMT",
+    #         "request_date_ms": "1587228396496",
+    #         "request_date_pst": "2020-04-18 09:46:36 America/Los_Angeles",
+    #         "original_purchase_date": "2013-08-01 07:00:00 Etc/GMT",
+    #         "original_purchase_date_ms": "1375340400000",
+    #         "original_purchase_date_pst": "2013-08-01 00:00:00 America/Los_Angeles",
+    #         "original_application_version": "1.0",
+    #         "in_app": [
+    #             {
+    #                 "quantity": "1",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #                 "transaction_id": "1000000653584474",
+    #                 "original_transaction_id": "1000000653584474",
+    #                 "purchase_date": "2020-04-18 16:27:42 Etc/GMT",
+    #                 "purchase_date_ms": "1587227262000",
+    #                 "purchase_date_pst": "2020-04-18 09:27:42 America/Los_Angeles",
+    #                 "original_purchase_date": "2020-04-18 16:27:44 Etc/GMT",
+    #                 "original_purchase_date_ms": "1587227264000",
+    #                 "original_purchase_date_pst": "2020-04-18 09:27:44 America/Los_Angeles",
+    #                 "expires_date": "2020-04-18 16:32:42 Etc/GMT",
+    #                 "expires_date_ms": "1587227562000",
+    #                 "expires_date_pst": "2020-04-18 09:32:42 America/Los_Angeles",
+    #                 "web_order_line_item_id": "1000000051847459",
+    #                 "is_trial_period": "false",
+    #                 "is_in_intro_offer_period": "false",
+    #             },
+    #             {
+    #                 "quantity": "1",
+    #                 "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #                 "transaction_id": "1000000653584861",
+    #                 "original_transaction_id": "1000000653584474",
+    #                 "purchase_date": "2020-04-18 16:32:42 Etc/GMT",
+    #                 "purchase_date_ms": "1587227562000",
+    #                 "purchase_date_pst": "2020-04-18 09:32:42 America/Los_Angeles",
+    #                 "original_purchase_date": "2020-04-18 16:27:44 Etc/GMT",
+    #                 "original_purchase_date_ms": "1587227264000",
+    #                 "original_purchase_date_pst": "2020-04-18 09:27:44 America/Los_Angeles",
+    #                 "expires_date": "2020-04-18 16:37:42 Etc/GMT",
+    #                 "expires_date_ms": "1587227862000",
+    #                 "expires_date_pst": "2020-04-18 09:37:42 America/Los_Angeles",
+    #                 "web_order_line_item_id": "1000000051847461",
+    #                 "is_trial_period": "false",
+    #                 "is_in_intro_offer_period": "false",
+    #             },
+    #         ],
+    #     },
+    #     "latest_receipt_info": [
+    #         {
+    #             "quantity": "1",
+    #             "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #             "transaction_id": "1000000653584474",
+    #             "original_transaction_id": "1000000653584474",
+    #             "purchase_date": "2020-04-18 16:27:42 Etc/GMT",
+    #             "purchase_date_ms": "1587227262000",
+    #             "purchase_date_pst": "2020-04-18 09:27:42 America/Los_Angeles",
+    #             "original_purchase_date": "2020-04-18 16:27:44 Etc/GMT",
+    #             "original_purchase_date_ms": "1587227264000",
+    #             "original_purchase_date_pst": "2020-04-18 09:27:44 America/Los_Angeles",
+    #             "expires_date": "2020-04-18 16:32:42 Etc/GMT",
+    #             "expires_date_ms": "1587227562000",
+    #             "expires_date_pst": "2020-04-18 09:32:42 America/Los_Angeles",
+    #             "web_order_line_item_id": "1000000051847459",
+    #             "is_trial_period": "false",
+    #             "is_in_intro_offer_period": "false",
+    #             "subscription_group_identifier": "20624274",
+    #         },
+    #         {
+    #             "quantity": "1",
+    #             "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #             "transaction_id": "1000000653584861",
+    #             "original_transaction_id": "1000000653584474",
+    #             "purchase_date": "2020-04-18 16:32:42 Etc/GMT",
+    #             "purchase_date_ms": "1587227562000",
+    #             "purchase_date_pst": "2020-04-18 09:32:42 America/Los_Angeles",
+    #             "original_purchase_date": "2020-04-18 16:27:44 Etc/GMT",
+    #             "original_purchase_date_ms": "1587227264000",
+    #             "original_purchase_date_pst": "2020-04-18 09:27:44 America/Los_Angeles",
+    #             "expires_date": "2020-04-18 16:37:42 Etc/GMT",
+    #             "expires_date_ms": "1587227862000",
+    #             "expires_date_pst": "2020-04-18 09:37:42 America/Los_Angeles",
+    #             "web_order_line_item_id": "1000000051847461",
+    #             "is_trial_period": "false",
+    #             "is_in_intro_offer_period": "false",
+    #             "subscription_group_identifier": "20624274",
+    #         },
+    #         {
+    #             "quantity": "1",
+    #             "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #             "transaction_id": "1000000653585235",
+    #             "original_transaction_id": "1000000653584474",
+    #             "purchase_date": "2020-04-18 16:38:16 Etc/GMT",
+    #             "purchase_date_ms": "1587227896000",
+    #             "purchase_date_pst": "2020-04-18 09:38:16 America/Los_Angeles",
+    #             "original_purchase_date": "2020-04-18 16:27:44 Etc/GMT",
+    #             "original_purchase_date_ms": "1587227264000",
+    #             "original_purchase_date_pst": "2020-04-18 09:27:44 America/Los_Angeles",
+    #             "expires_date": "2020-04-18 16:43:16 Etc/GMT",
+    #             "expires_date_ms": "1587228196000",
+    #             "expires_date_pst": "2020-04-18 09:43:16 America/Los_Angeles",
+    #             "web_order_line_item_id": "1000000051847500",
+    #             "is_trial_period": "false",
+    #             "is_in_intro_offer_period": "false",
+    #             "subscription_group_identifier": "20624274",
+    #         },
+    #         {
+    #             "quantity": "1",
+    #             "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #             "transaction_id": "1000000653585760",
+    #             "original_transaction_id": "1000000653584474",
+    #             "purchase_date": "2020-04-18 16:44:25 Etc/GMT",
+    #             "purchase_date_ms": "1587228265000",
+    #             "purchase_date_pst": "2020-04-18 09:44:25 America/Los_Angeles",
+    #             "original_purchase_date": "2020-04-18 16:27:44 Etc/GMT",
+    #             "original_purchase_date_ms": "1587227264000",
+    #             "original_purchase_date_pst": "2020-04-18 09:27:44 America/Los_Angeles",
+    #             "expires_date": "2020-04-18 16:49:25 Etc/GMT",
+    #             "expires_date_ms": "1587228565000",
+    #             "expires_date_pst": "2020-04-18 09:49:25 America/Los_Angeles",
+    #             "web_order_line_item_id": "1000000051847566",
+    #             "is_trial_period": "false",
+    #             "is_in_intro_offer_period": "false",
+    #             "subscription_group_identifier": "20624274",
+    #         },
+    #     ],
+    #     "latest_receipt": "very long string",
+    #     "pending_renewal_info": [
+    #         {
+    #             "auto_renew_product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #             "original_transaction_id": "1000000653584474",
+    #             "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #             "auto_renew_status": "1",
+    #         }
+    #     ],
+    # }
+
+    if data["status"] != 0:
+        LOG.e(
+            "verifyReceipt status !=0, probably invalid receipt. User %s, data %s",
+            user,
+            data,
+        )
+        return None
+
+    # use responseBody.Latest_receipt_info and not responseBody.Receipt.In_app
+    # as recommended on https://developer.apple.com/documentation/appstorereceipts/responsebody/receipt/in_app
+    # each item in data["latest_receipt_info"] has the following format
+    # {
+    #     "quantity": "1",
+    #     "product_id": "io.simplelogin.ios_app.subscription.premium.monthly",
+    #     "transaction_id": "1000000653584474",
+    #     "original_transaction_id": "1000000653584474",
+    #     "purchase_date": "2020-04-18 16:27:42 Etc/GMT",
+    #     "purchase_date_ms": "1587227262000",
+    #     "purchase_date_pst": "2020-04-18 09:27:42 America/Los_Angeles",
+    #     "original_purchase_date": "2020-04-18 16:27:44 Etc/GMT",
+    #     "original_purchase_date_ms": "1587227264000",
+    #     "original_purchase_date_pst": "2020-04-18 09:27:44 America/Los_Angeles",
+    #     "expires_date": "2020-04-18 16:32:42 Etc/GMT",
+    #     "expires_date_ms": "1587227562000",
+    #     "expires_date_pst": "2020-04-18 09:32:42 America/Los_Angeles",
+    #     "web_order_line_item_id": "1000000051847459",
+    #     "is_trial_period": "false",
+    #     "is_in_intro_offer_period": "false",
+    # }
+    transactions = data.get("latest_receipt_info")
+    if not transactions:
+        LOG.i("Empty transactions in data %s", data)
+        return None
+
+    latest_transaction = max(transactions, key=lambda t: int(t["expires_date_ms"]))
+    original_transaction_id = latest_transaction["original_transaction_id"]
+    expires_date = arrow.get(int(latest_transaction["expires_date_ms"]) / 1000)
+    plan = (
+        PlanEnum.monthly
+        if latest_transaction["product_id"]
+        in (
+            _MONTHLY_PRODUCT_ID,
+            _MACAPP_MONTHLY_PRODUCT_ID,
+            _MACAPP_MONTHLY_PRODUCT_ID_NEW,
+        )
+        else PlanEnum.yearly
+    )
+
+    apple_sub: AppleSubscription = AppleSubscription.get_by(user_id=user.id)
+
+    if apple_sub:
+        LOG.d(
+            "Update AppleSubscription for user %s, expired at %s (%s), plan %s",
+            user,
+            expires_date,
+            expires_date.humanize(),
+            plan,
+        )
+        apple_sub.receipt_data = receipt_data
+        apple_sub.expires_date = expires_date
+        apple_sub.original_transaction_id = original_transaction_id
+        apple_sub.product_id = latest_transaction["product_id"]
+        apple_sub.plan = plan
+    else:
+        # the same original_transaction_id has been used on another account
+        if AppleSubscription.get_by(original_transaction_id=original_transaction_id):
+            LOG.e("Same Apple Sub has been used before, current user %s", user)
+            return None
+
+        LOG.d(
+            "Create new AppleSubscription for user %s, expired at %s, plan %s",
+            user,
+            expires_date,
+            plan,
+        )
+        apple_sub = AppleSubscription.create(
+            user_id=user.id,
+            receipt_data=receipt_data,
+            expires_date=expires_date,
+            original_transaction_id=original_transaction_id,
+            plan=plan,
+            product_id=latest_transaction["product_id"],
+        )
+
+    execute_subscription_webhook(user)
+    Session.commit()
+
+    return apple_sub

app/api/views/auth.py

@@ -1,28 +1,33 @@
-import random
+import secrets
+import string
 
 import facebook
 import google.oauth2.credentials
 import googleapiclient.discovery
 from flask import jsonify, request
-from flask_cors import cross_origin
+from flask_login import login_user
 from itsdangerous import Signer
 
 from app import email_utils
 from app.api.base import api_bp
 from app.config import FLASK_SECRET, DISABLE_REGISTRATION
+from app.dashboard.views.account_setting import send_reset_password_email
+from app.db import Session
 from app.email_utils import (
-    can_be_used_as_personal_email,
-    email_already_used,
+    email_can_be_used_as_mailbox,
+    personal_email_already_used,
     send_email,
     render,
 )
-from app.extensions import db
+from app.events.auth_event import LoginEvent, RegisterEvent
+from app.extensions import limiter
 from app.log import LOG
 from app.models import User, ApiKey, SocialAuth, AccountActivation
+from app.utils import sanitize_email, canonicalize_email
 
 
 @api_bp.route("/auth/login", methods=["POST"])
-@cross_origin()
+@limiter.limit("10/minute")
 def auth_login():
     """
     Authenticate user
@@ -44,22 +49,39 @@ def auth_login():
     if not data:
         return jsonify(error="request body cannot be empty"), 400
 
-    email = data.get("email")
     password = data.get("password")
     device = data.get("device")
 
-    user = User.filter_by(email=email).first()
+    email = sanitize_email(data.get("email"))
+    canonical_email = canonicalize_email(data.get("email"))
+
+    user = User.get_by(email=email) or User.get_by(email=canonical_email)
 
     if not user or not user.check_password(password):
+        LoginEvent(LoginEvent.ActionType.failed, LoginEvent.Source.api).send()
         return jsonify(error="Email or password incorrect"), 400
+    elif user.disabled:
+        LoginEvent(LoginEvent.ActionType.disabled_login, LoginEvent.Source.api).send()
+        return jsonify(error="Account disabled"), 400
+    elif user.delete_on is not None:
+        LoginEvent(
+            LoginEvent.ActionType.scheduled_to_be_deleted, LoginEvent.Source.api
+        ).send()
+        return jsonify(error="Account scheduled for deletion"), 400
     elif not user.activated:
-        return jsonify(error="Account not activated"), 400
+        LoginEvent(LoginEvent.ActionType.not_activated, LoginEvent.Source.api).send()
+        return jsonify(error="Account not activated"), 422
+    elif user.fido_enabled():
+        # allow user who has TOTP enabled to continue using the mobile app
+        if not user.enable_otp:
+            return jsonify(error="Currently we don't support FIDO on mobile yet"), 403
 
+    LoginEvent(LoginEvent.ActionType.success, LoginEvent.Source.api).send()
     return jsonify(**auth_payload(user, device)), 200
 
 
 @api_bp.route("/auth/register", methods=["POST"])
-@cross_origin()
+@limiter.limit("10/minute")
 def auth_register():
     """
     User signs up - will need to activate their account with an activation code.
@@ -74,38 +96,49 @@ def auth_register():
     if not data:
         return jsonify(error="request body cannot be empty"), 400
 
-    email = data.get("email")
+    dirty_email = data.get("email")
+    email = canonicalize_email(dirty_email)
     password = data.get("password")
 
     if DISABLE_REGISTRATION:
+        RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
         return jsonify(error="registration is closed"), 400
-    if not can_be_used_as_personal_email(email) or email_already_used(email):
+    if not email_can_be_used_as_mailbox(email) or personal_email_already_used(email):
+        RegisterEvent(
+            RegisterEvent.ActionType.invalid_email, RegisterEvent.Source.api
+        ).send()
         return jsonify(error=f"cannot use {email} as personal inbox"), 400
 
     if not password or len(password) < 8:
+        RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
         return jsonify(error="password too short"), 400
 
-    LOG.debug("create user %s", email)
-    user = User.create(email=email, name="", password=password)
-    db.session.flush()
+    if len(password) > 100:
+        RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
+        return jsonify(error="password too long"), 400
+
+    LOG.d("create user %s", email)
+    user = User.create(email=email, name=dirty_email, password=password)
+    Session.flush()
 
     # create activation code
-    code = "".join([str(random.randint(0, 9)) for _ in range(6)])
+    code = "".join([str(secrets.choice(string.digits)) for _ in range(6)])
     AccountActivation.create(user_id=user.id, code=code)
-    db.session.commit()
+    Session.commit()
 
     send_email(
         email,
-        f"Just one more step to join SimpleLogin",
-        render("transactional/code-activation.txt", code=code),
+        "Just one more step to join SimpleLogin",
+        render("transactional/code-activation.txt.jinja2", code=code),
         render("transactional/code-activation.html", code=code),
     )
 
+    RegisterEvent(RegisterEvent.ActionType.success, RegisterEvent.Source.api).send()
     return jsonify(msg="User needs to confirm their account"), 200
 
 
 @api_bp.route("/auth/activate", methods=["POST"])
-@cross_origin()
+@limiter.limit("10/minute")
 def auth_activate():
     """
     User enters the activation code to confirm their account.
@@ -122,10 +155,11 @@ def auth_activate():
     if not data:
         return jsonify(error="request body cannot be empty"), 400
 
-    email = data.get("email")
+    email = sanitize_email(data.get("email"))
+    canonical_email = canonicalize_email(data.get("email"))
     code = data.get("code")
 
-    user = User.get_by(email=email)
+    user = User.get_by(email=email) or User.get_by(email=canonical_email)
 
     # do not use a different message to avoid exposing existing email
     if not user or user.activated:
@@ -138,25 +172,25 @@ def auth_activate():
     if account_activation.code != code:
         # decrement nb tries
         account_activation.tries -= 1
-        db.session.commit()
+        Session.commit()
 
         if account_activation.tries == 0:
             AccountActivation.delete(account_activation.id)
-            db.session.commit()
+            Session.commit()
             return jsonify(error="Too many wrong tries"), 410
 
         return jsonify(error="Wrong email or code"), 400
 
-    LOG.debug("activate user %s", user)
+    LOG.d("activate user %s", user)
     user.activated = True
     AccountActivation.delete(account_activation.id)
-    db.session.commit()
+    Session.commit()
 
     return jsonify(msg="Account is activated, user can login now"), 200
 
 
 @api_bp.route("/auth/reactivate", methods=["POST"])
-@cross_origin()
+@limiter.limit("10/minute")
 def auth_reactivate():
     """
     User asks for another activation code
@@ -170,8 +204,10 @@ def auth_reactivate():
     if not data:
         return jsonify(error="request body cannot be empty"), 400
 
-    email = data.get("email")
-    user = User.get_by(email=email)
+    email = sanitize_email(data.get("email"))
+    canonical_email = canonicalize_email(data.get("email"))
+
+    user = User.get_by(email=email) or User.get_by(email=canonical_email)
 
     # do not use a different message to avoid exposing existing email
     if not user or user.activated:
@@ -180,17 +216,17 @@ def auth_reactivate():
     account_activation = AccountActivation.get_by(user_id=user.id)
     if account_activation:
         AccountActivation.delete(account_activation.id)
-        db.session.commit()
+        Session.commit()
 
     # create activation code
-    code = "".join([str(random.randint(0, 9)) for _ in range(6)])
+    code = "".join([str(secrets.choice(string.digits)) for _ in range(6)])
     AccountActivation.create(user_id=user.id, code=code)
-    db.session.commit()
+    Session.commit()
 
     send_email(
         email,
-        f"Just one more step to join SimpleLogin",
-        render("transactional/code-activation.txt", code=code),
+        "Just one more step to join SimpleLogin",
+        render("transactional/code-activation.txt.jinja2", code=code),
         render("transactional/code-activation.html", code=code),
     )
 
@@ -198,7 +234,7 @@ def auth_reactivate():
 
 
 @api_bp.route("/auth/facebook", methods=["POST"])
-@cross_origin()
+@limiter.limit("10/minute")
 def auth_facebook():
     """
     Authenticate user with Facebook
@@ -224,33 +260,35 @@ def auth_facebook():
 
     graph = facebook.GraphAPI(access_token=facebook_token)
     user_info = graph.get_object("me", fields="email,name")
-    email = user_info.get("email")
+    email = sanitize_email(user_info.get("email"))
 
     user = User.get_by(email=email)
 
     if not user:
         if DISABLE_REGISTRATION:
             return jsonify(error="registration is closed"), 400
-        if not can_be_used_as_personal_email(email) or email_already_used(email):
+        if not email_can_be_used_as_mailbox(email) or personal_email_already_used(
+            email
+        ):
             return jsonify(error=f"cannot use {email} as personal inbox"), 400
 
         LOG.d("create facebook user with %s", user_info)
-        user = User.create(email=email.lower(), name=user_info["name"], activated=True)
-        db.session.commit()
+        user = User.create(email=email, name=user_info["name"], activated=True)
+        Session.commit()
         email_utils.send_welcome_email(user)
 
     if not SocialAuth.get_by(user_id=user.id, social="facebook"):
         SocialAuth.create(user_id=user.id, social="facebook")
-        db.session.commit()
+        Session.commit()
 
     return jsonify(**auth_payload(user, device)), 200
 
 
 @api_bp.route("/auth/google", methods=["POST"])
-@cross_origin()
+@limiter.limit("10/minute")
 def auth_google():
     """
-    Authenticate user with Facebook
+    Authenticate user with Google
     Input:
         google_token: Google access token
         device: to create an ApiKey associated with this device
@@ -276,30 +314,32 @@ def auth_google():
     build = googleapiclient.discovery.build("oauth2", "v2", credentials=cred)
 
     user_info = build.userinfo().get().execute()
-    email = user_info.get("email")
+    email = sanitize_email(user_info.get("email"))
 
     user = User.get_by(email=email)
 
     if not user:
         if DISABLE_REGISTRATION:
             return jsonify(error="registration is closed"), 400
-        if not can_be_used_as_personal_email(email) or email_already_used(email):
+        if not email_can_be_used_as_mailbox(email) or personal_email_already_used(
+            email
+        ):
             return jsonify(error=f"cannot use {email} as personal inbox"), 400
 
         LOG.d("create Google user with %s", user_info)
-        user = User.create(email=email.lower(), name="", activated=True)
-        db.session.commit()
+        user = User.create(email=email, name="", activated=True)
+        Session.commit()
         email_utils.send_welcome_email(user)
 
     if not SocialAuth.get_by(user_id=user.id, social="google"):
         SocialAuth.create(user_id=user.id, social="google")
-        db.session.commit()
+        Session.commit()
 
     return jsonify(**auth_payload(user, device)), 200
 
 
 def auth_payload(user, device) -> dict:
-    ret = {"name": user.name, "mfa_enabled": user.enable_otp}
+    ret = {"name": user.name or "", "email": user.email, "mfa_enabled": user.enable_otp}
 
     # do not give api_key, user can only obtain api_key after OTP verification
     if user.enable_otp:
@@ -311,8 +351,38 @@ def auth_payload(user, device) -> dict:
         if not api_key:
             LOG.d("create new api key for %s and %s", user, device)
             api_key = ApiKey.create(user.id, device)
-            db.session.commit()
+            Session.commit()
         ret["mfa_key"] = None
         ret["api_key"] = api_key.code
 
+        # so user is automatically logged in on the web
+        login_user(user)
+
     return ret
+
+
+@api_bp.route("/auth/forgot_password", methods=["POST"])
+@limiter.limit("2/minute")
+def forgot_password():
+    """
+    User forgot password
+    Input:
+        email
+    Output:
+        200 and a reset password email is sent to user
+        400 if email not exist
+
+    """
+    data = request.get_json()
+    if not data or not data.get("email"):
+        return jsonify(error="request body must contain email"), 400
+
+    email = sanitize_email(data.get("email"))
+    canonical_email = canonicalize_email(data.get("email"))
+
+    user = User.get_by(email=email) or User.get_by(email=canonical_email)
+
+    if user:
+        send_reset_password_email(user)
+
+    return jsonify(ok=True)

app/api/views/auth_mfa.py

@@ -1,17 +1,19 @@
 import pyotp
 from flask import jsonify, request
-from flask_cors import cross_origin
-from itsdangerous import Signer, BadSignature
+from flask_login import login_user
+from itsdangerous import Signer
 
 from app.api.base import api_bp
 from app.config import FLASK_SECRET
-from app.extensions import db
+from app.db import Session
+from app.email_utils import send_invalid_totp_login_email
+from app.extensions import limiter
 from app.log import LOG
 from app.models import User, ApiKey
 
 
 @api_bp.route("/auth/mfa", methods=["POST"])
-@cross_origin()
+@limiter.limit("10/minute")
 def auth_mfa():
     """
     Validate the OTP Token
@@ -23,7 +25,8 @@ def auth_mfa():
         200 and user info containing:
         {
             name: "John Wick",
-            api_key: "a long string"
+            api_key: "a long string",
+            email: "user email"
         }
 
     """
@@ -52,17 +55,21 @@ def auth_mfa():
         )
 
     totp = pyotp.TOTP(user.otp_secret)
-    if not totp.verify(mfa_token):
+    if not totp.verify(mfa_token, valid_window=2):
+        send_invalid_totp_login_email(user, "TOTP")
         return jsonify(error="Wrong TOTP Token"), 400
 
-    ret = {"name": user.name}
+    ret = {"name": user.name or "", "email": user.email}
 
     api_key = ApiKey.get_by(user_id=user.id, name=device)
     if not api_key:
         LOG.d("create new api key for %s and %s", user, device)
         api_key = ApiKey.create(user.id, device)
-        db.session.commit()
+        Session.commit()
 
     ret["api_key"] = api_key.code
 
+    # so user is logged in automatically on the web
+    login_user(user)
+
     return jsonify(**ret), 200

app/api/views/custom_domain.py

@@ -0,0 +1,126 @@
+from flask import g, request
+from flask import jsonify
+
+from app.api.base import api_bp, require_api_auth
+from app.db import Session
+from app.models import CustomDomain, DomainDeletedAlias, Mailbox, DomainMailbox
+
+
+def custom_domain_to_dict(custom_domain: CustomDomain):
+    return {
+        "id": custom_domain.id,
+        "domain_name": custom_domain.domain,
+        "is_verified": custom_domain.verified,
+        "nb_alias": custom_domain.nb_alias(),
+        "creation_date": custom_domain.created_at.format(),
+        "creation_timestamp": custom_domain.created_at.timestamp,
+        "catch_all": custom_domain.catch_all,
+        "name": custom_domain.name,
+        "random_prefix_generation": custom_domain.random_prefix_generation,
+        "mailboxes": [
+            {"id": mb.id, "email": mb.email} for mb in custom_domain.mailboxes
+        ],
+    }
+
+
+@api_bp.route("/custom_domains", methods=["GET"])
+@require_api_auth
+def get_custom_domains():
+    user = g.user
+    custom_domains = CustomDomain.filter_by(
+        user_id=user.id, is_sl_subdomain=False
+    ).all()
+
+    return jsonify(custom_domains=[custom_domain_to_dict(cd) for cd in custom_domains])
+
+
+@api_bp.route("/custom_domains/<int:custom_domain_id>/trash", methods=["GET"])
+@require_api_auth
+def get_custom_domain_trash(custom_domain_id: int):
+    user = g.user
+    custom_domain = CustomDomain.get(custom_domain_id)
+    if not custom_domain or custom_domain.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    domain_deleted_aliases = DomainDeletedAlias.filter_by(
+        domain_id=custom_domain.id
+    ).all()
+
+    return jsonify(
+        aliases=[
+            {
+                "alias": dda.email,
+                "deletion_timestamp": dda.created_at.timestamp,
+            }
+            for dda in domain_deleted_aliases
+        ]
+    )
+
+
+@api_bp.route("/custom_domains/<int:custom_domain_id>", methods=["PATCH"])
+@require_api_auth
+def update_custom_domain(custom_domain_id):
+    """
+    Update alias note
+    Input:
+        custom_domain_id: in url
+    In body:
+        catch_all (optional): boolean
+        random_prefix_generation (optional): boolean
+        name (optional): in body
+        mailbox_ids (optional): array of mailbox_id
+    Output:
+        200
+    """
+    data = request.get_json()
+    if not data:
+        return jsonify(error="request body cannot be empty"), 400
+
+    user = g.user
+    custom_domain: CustomDomain = CustomDomain.get(custom_domain_id)
+
+    if not custom_domain or custom_domain.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    changed = False
+    if "catch_all" in data:
+        catch_all = data.get("catch_all")
+        custom_domain.catch_all = catch_all
+        changed = True
+
+    if "random_prefix_generation" in data:
+        random_prefix_generation = data.get("random_prefix_generation")
+        custom_domain.random_prefix_generation = random_prefix_generation
+        changed = True
+
+    if "name" in data:
+        name = data.get("name")
+        custom_domain.name = name
+        changed = True
+
+    if "mailbox_ids" in data:
+        mailbox_ids = [int(m_id) for m_id in data.get("mailbox_ids")]
+        if mailbox_ids:
+            # check if mailbox is not tempered with
+            mailboxes = []
+            for mailbox_id in mailbox_ids:
+                mailbox = Mailbox.get(mailbox_id)
+                if not mailbox or mailbox.user_id != user.id or not mailbox.verified:
+                    return jsonify(error="Forbidden"), 400
+                mailboxes.append(mailbox)
+
+            # first remove all existing domain-mailboxes links
+            DomainMailbox.filter_by(domain_id=custom_domain.id).delete()
+            Session.flush()
+
+            for mailbox in mailboxes:
+                DomainMailbox.create(domain_id=custom_domain.id, mailbox_id=mailbox.id)
+
+            changed = True
+
+    if changed:
+        Session.commit()
+
+    # refresh
+    custom_domain = CustomDomain.get(custom_domain_id)
+    return jsonify(custom_domain=custom_domain_to_dict(custom_domain)), 200

app/api/views/export.py

@@ -0,0 +1,49 @@
+from flask import g
+from flask import jsonify
+
+from app.api.base import api_bp, require_api_auth
+from app.models import Alias, Client, CustomDomain
+from app.alias_utils import alias_export_csv
+
+
+@api_bp.route("/export/data", methods=["GET"])
+@require_api_auth
+def export_data():
+    """
+    Get user data
+    Output:
+        Alias, custom domain and app info
+
+    """
+    user = g.user
+
+    data = {
+        "email": user.email,
+        "name": user.name,
+        "aliases": [],
+        "apps": [],
+        "custom_domains": [],
+    }
+
+    for alias in Alias.filter_by(user_id=user.id).all():  # type: Alias
+        data["aliases"].append(dict(email=alias.email, enabled=alias.enabled))
+
+    for custom_domain in CustomDomain.filter_by(user_id=user.id).all():
+        data["custom_domains"].append(custom_domain.domain)
+
+    for app in Client.filter_by(user_id=user.id):  # type: Client
+        data["apps"].append(dict(name=app.name, home_url=app.home_url))
+
+    return jsonify(data)
+
+
+@api_bp.route("/export/aliases", methods=["GET"])
+@require_api_auth
+def export_aliases():
+    """
+    Get user aliases as importable CSV file
+    Output:
+        Importable CSV file
+
+    """
+    return alias_export_csv(g.user)

app/api/views/mailbox.py

@@ -0,0 +1,236 @@
+from smtplib import SMTPRecipientsRefused
+
+import arrow
+from flask import g
+from flask import jsonify
+from flask import request
+
+from app.api.base import api_bp, require_api_auth
+from app.config import JOB_DELETE_MAILBOX
+from app.dashboard.views.mailbox import send_verification_email
+from app.dashboard.views.mailbox_detail import verify_mailbox_change
+from app.db import Session
+from app.email_utils import (
+    mailbox_already_used,
+    email_can_be_used_as_mailbox,
+)
+from app.email_validation import is_valid_email
+from app.log import LOG
+from app.models import Mailbox, Job
+from app.utils import sanitize_email
+
+
+def mailbox_to_dict(mailbox: Mailbox):
+    return {
+        "id": mailbox.id,
+        "email": mailbox.email,
+        "verified": mailbox.verified,
+        "default": mailbox.user.default_mailbox_id == mailbox.id,
+        "creation_timestamp": mailbox.created_at.timestamp,
+        "nb_alias": mailbox.nb_alias(),
+    }
+
+
+@api_bp.route("/mailboxes", methods=["POST"])
+@require_api_auth
+def create_mailbox():
+    """
+    Create a new mailbox. User needs to verify the mailbox via an activation email.
+    Input:
+        email: in body
+    Output:
+        the new mailbox dict
+    """
+    user = g.user
+    mailbox_email = sanitize_email(request.get_json().get("email"))
+
+    if not user.is_premium():
+        return jsonify(error="Only premium plan can add additional mailbox"), 400
+
+    if not is_valid_email(mailbox_email):
+        return jsonify(error=f"{mailbox_email} invalid"), 400
+    elif mailbox_already_used(mailbox_email, user):
+        return jsonify(error=f"{mailbox_email} already used"), 400
+    elif not email_can_be_used_as_mailbox(mailbox_email):
+        return (
+            jsonify(
+                error=f"{mailbox_email} cannot be used. Please note a mailbox cannot "
+                f"be a disposable email address"
+            ),
+            400,
+        )
+    else:
+        new_mailbox = Mailbox.create(email=mailbox_email, user_id=user.id)
+        Session.commit()
+
+        send_verification_email(user, new_mailbox)
+
+        return (
+            jsonify(mailbox_to_dict(new_mailbox)),
+            201,
+        )
+
+
+@api_bp.route("/mailboxes/<int:mailbox_id>", methods=["DELETE"])
+@require_api_auth
+def delete_mailbox(mailbox_id):
+    """
+    Delete mailbox
+    Input:
+        mailbox_id: in url
+        (optional) transfer_aliases_to: in body. Id of the new mailbox for the aliases.
+                                        If omitted or the value is set to -1,
+                                        the aliases of the mailbox will be deleted too.
+    Output:
+        200 if deleted successfully
+
+    """
+    user = g.user
+    mailbox = Mailbox.get(mailbox_id)
+
+    if not mailbox or mailbox.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    if mailbox.id == user.default_mailbox_id:
+        return jsonify(error="You cannot delete the default mailbox"), 400
+
+    data = request.get_json() or {}
+    transfer_mailbox_id = data.get("transfer_aliases_to")
+    if transfer_mailbox_id and int(transfer_mailbox_id) >= 0:
+        transfer_mailbox = Mailbox.get(transfer_mailbox_id)
+
+        if not transfer_mailbox or transfer_mailbox.user_id != user.id:
+            return (
+                jsonify(error="You must transfer the aliases to a mailbox you own."),
+                403,
+            )
+
+        if transfer_mailbox_id == mailbox_id:
+            return (
+                jsonify(
+                    error="You can not transfer the aliases to the mailbox you want to delete."
+                ),
+                400,
+            )
+
+        if not transfer_mailbox.verified:
+            return jsonify(error="Your new mailbox is not verified"), 400
+
+    # Schedule delete account job
+    LOG.w("schedule delete mailbox job for %s", mailbox)
+    Job.create(
+        name=JOB_DELETE_MAILBOX,
+        payload={
+            "mailbox_id": mailbox.id,
+            "transfer_mailbox_id": transfer_mailbox_id,
+        },
+        run_at=arrow.now(),
+        commit=True,
+    )
+
+    return jsonify(deleted=True), 200
+
+
+@api_bp.route("/mailboxes/<int:mailbox_id>", methods=["PUT"])
+@require_api_auth
+def update_mailbox(mailbox_id):
+    """
+    Update mailbox
+    Input:
+        mailbox_id: in url
+        (optional) default: in body. Set a mailbox as the default mailbox.
+        (optional) email: in body. Change a mailbox email.
+        (optional) cancel_email_change: in body. Cancel mailbox email change.
+    Output:
+        200 if updated successfully
+
+    """
+    user = g.user
+    mailbox = Mailbox.get(mailbox_id)
+
+    if not mailbox or mailbox.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    data = request.get_json() or {}
+    changed = False
+    if "default" in data:
+        is_default = data.get("default")
+        if is_default:
+            if not mailbox.verified:
+                return (
+                    jsonify(
+                        error="Unverified mailbox cannot be used as default mailbox"
+                    ),
+                    400,
+                )
+            user.default_mailbox_id = mailbox.id
+            changed = True
+
+    if "email" in data:
+        new_email = sanitize_email(data.get("email"))
+
+        if mailbox_already_used(new_email, user):
+            return jsonify(error=f"{new_email} already used"), 400
+        elif not email_can_be_used_as_mailbox(new_email):
+            return (
+                jsonify(
+                    error=f"{new_email} cannot be used. Please note a mailbox cannot "
+                    f"be a disposable email address"
+                ),
+                400,
+            )
+
+        try:
+            verify_mailbox_change(user, mailbox, new_email)
+        except SMTPRecipientsRefused:
+            return jsonify(error=f"Incorrect mailbox, please recheck {new_email}"), 400
+        else:
+            mailbox.new_email = new_email
+            changed = True
+
+    if "cancel_email_change" in data:
+        cancel_email_change = data.get("cancel_email_change")
+        if cancel_email_change:
+            mailbox.new_email = None
+            changed = True
+
+    if changed:
+        Session.commit()
+
+    return jsonify(updated=True), 200
+
+
+@api_bp.route("/mailboxes", methods=["GET"])
+@require_api_auth
+def get_mailboxes():
+    """
+    Get verified mailboxes
+    Output:
+        - mailboxes: list of mailbox dict
+    """
+    user = g.user
+
+    return (
+        jsonify(mailboxes=[mailbox_to_dict(mb) for mb in user.mailboxes()]),
+        200,
+    )
+
+
+@api_bp.route("/v2/mailboxes", methods=["GET"])
+@require_api_auth
+def get_mailboxes_v2():
+    """
+    Get all mailboxes - including unverified mailboxes
+    Output:
+        - mailboxes: list of mailbox dict
+    """
+    user = g.user
+    mailboxes = []
+
+    for mailbox in Mailbox.filter_by(user_id=user.id):
+        mailboxes.append(mailbox)
+
+    return (
+        jsonify(mailboxes=[mailbox_to_dict(mb) for mb in mailboxes]),
+        200,
+    )

app/api/views/new_custom_alias.py

@@ -1,25 +1,42 @@
 from flask import g
 from flask import jsonify, request
-from flask_cors import cross_origin
 
-from app.api.base import api_bp, verify_api_key
-from app.config import MAX_NB_EMAIL_FREE_PLAN
-from app.dashboard.views.custom_alias import verify_prefix_suffix
-from app.extensions import db
+from app import parallel_limiter
+from app.alias_suffix import check_suffix_signature, verify_prefix_suffix
+from app.alias_utils import check_alias_prefix
+from app.api.base import api_bp, require_api_auth
+from app.api.serializer import (
+    serialize_alias_info_v2,
+    get_alias_info_v2,
+)
+from app.config import MAX_NB_EMAIL_FREE_PLAN, ALIAS_LIMIT
+from app.db import Session
+from app.extensions import limiter
 from app.log import LOG
-from app.models import GenEmail, AliasUsedOn, User
+from app.models import (
+    Alias,
+    AliasUsedOn,
+    User,
+    DeletedAlias,
+    DomainDeletedAlias,
+    Mailbox,
+    AliasMailbox,
+)
 from app.utils import convert_to_id
 
 
-@api_bp.route("/alias/custom/new", methods=["POST"])
-@cross_origin()
-@verify_api_key
-def new_custom_alias():
+@api_bp.route("/v2/alias/custom/new", methods=["POST"])
+@limiter.limit(ALIAS_LIMIT)
+@require_api_auth
+@parallel_limiter.lock(name="alias_creation")
+def new_custom_alias_v2():
     """
     Create a new custom alias
+    Same as v1 but signed_suffix is actually the suffix with signature, e.g.
+    .random_word@SL.co.Xq19rQ.s99uWQ7jD1s5JZDZqczYI5TbNNU
     Input:
         alias_prefix, for ex "www_groupon_com"
-        alias_suffix, either .random_letters@simplelogin.co or @my-domain.com
+        signed_suffix, either .random_letters@simplelogin.co or @my-domain.com
         optional "hostname" in args
         optional "note"
     Output:
@@ -38,33 +55,181 @@ def new_custom_alias():
             400,
         )
 
-    user_custom_domains = [cd.domain for cd in user.verified_custom_domains()]
     hostname = request.args.get("hostname")
 
     data = request.get_json()
     if not data:
         return jsonify(error="request body cannot be empty"), 400
 
-    alias_prefix = data.get("alias_prefix", "").strip()
-    alias_suffix = data.get("alias_suffix", "").strip()
+    alias_prefix = data.get("alias_prefix", "").strip().lower().replace(" ", "")
+    signed_suffix = data.get("signed_suffix", "").strip()
     note = data.get("note")
     alias_prefix = convert_to_id(alias_prefix)
 
-    if not verify_prefix_suffix(user, alias_prefix, alias_suffix, user_custom_domains):
+    try:
+        alias_suffix = check_suffix_signature(signed_suffix)
+        if not alias_suffix:
+            LOG.w("Alias creation time expired for %s", user)
+            return jsonify(error="Alias creation time is expired, please retry"), 412
+    except Exception:
+        LOG.w("Alias suffix is tampered, user %s", user)
+        return jsonify(error="Tampered suffix"), 400
+
+    if not verify_prefix_suffix(user, alias_prefix, alias_suffix):
         return jsonify(error="wrong alias prefix or suffix"), 400
 
     full_alias = alias_prefix + alias_suffix
-    if GenEmail.get_by(email=full_alias):
+    if (
+        Alias.get_by(email=full_alias)
+        or DeletedAlias.get_by(email=full_alias)
+        or DomainDeletedAlias.get_by(email=full_alias)
+    ):
         LOG.d("full alias already used %s", full_alias)
         return jsonify(error=f"alias {full_alias} already exists"), 409
 
-    gen_email = GenEmail.create(
-        user_id=user.id, email=full_alias, mailbox_id=user.default_mailbox_id, note=note
+    if ".." in full_alias:
+        return (
+            jsonify(error="2 consecutive dot signs aren't allowed in an email address"),
+            400,
+        )
+
+    alias = Alias.create(
+        user_id=user.id,
+        email=full_alias,
+        mailbox_id=user.default_mailbox_id,
+        note=note,
     )
-    db.session.commit()
+
+    Session.commit()
 
     if hostname:
-        AliasUsedOn.create(gen_email_id=gen_email.id, hostname=hostname)
-        db.session.commit()
+        AliasUsedOn.create(alias_id=alias.id, hostname=hostname, user_id=alias.user_id)
+        Session.commit()
 
-    return jsonify(alias=full_alias), 201
+    return (
+        jsonify(alias=full_alias, **serialize_alias_info_v2(get_alias_info_v2(alias))),
+        201,
+    )
+
+
+@api_bp.route("/v3/alias/custom/new", methods=["POST"])
+@limiter.limit(ALIAS_LIMIT)
+@require_api_auth
+@parallel_limiter.lock(name="alias_creation")
+def new_custom_alias_v3():
+    """
+    Create a new custom alias
+    Same as v2 but accept a list of mailboxes as input
+    Input:
+        alias_prefix, for ex "www_groupon_com"
+        signed_suffix, either .random_letters@simplelogin.co or @my-domain.com
+        mailbox_ids: list of int
+        optional "hostname" in args
+        optional "note"
+        optional "name"
+
+    Output:
+        201 if success
+        409 if the alias already exists
+
+    """
+    user: User = g.user
+    if not user.can_create_new_alias():
+        LOG.d("user %s cannot create any custom alias", user)
+        return (
+            jsonify(
+                error="You have reached the limitation of a free account with the maximum of "
+                f"{MAX_NB_EMAIL_FREE_PLAN} aliases, please upgrade your plan to create more aliases"
+            ),
+            400,
+        )
+
+    hostname = request.args.get("hostname")
+
+    data = request.get_json()
+    if not data:
+        return jsonify(error="request body cannot be empty"), 400
+
+    if not isinstance(data, dict):
+        return jsonify(error="request body does not follow the required format"), 400
+
+    alias_prefix = data.get("alias_prefix", "").strip().lower().replace(" ", "")
+    signed_suffix = data.get("signed_suffix", "") or ""
+    signed_suffix = signed_suffix.strip()
+
+    mailbox_ids = data.get("mailbox_ids")
+    note = data.get("note")
+    name = data.get("name")
+    if name:
+        name = name.replace("\n", "")
+    alias_prefix = convert_to_id(alias_prefix)
+
+    if not check_alias_prefix(alias_prefix):
+        return jsonify(error="alias prefix invalid format or too long"), 400
+
+    # check if mailbox is not tempered with
+    if not isinstance(mailbox_ids, list):
+        return jsonify(error="mailbox_ids must be an array of id"), 400
+    mailboxes = []
+    for mailbox_id in mailbox_ids:
+        mailbox = Mailbox.get(mailbox_id)
+        if not mailbox or mailbox.user_id != user.id or not mailbox.verified:
+            return jsonify(error="Errors with Mailbox"), 400
+        mailboxes.append(mailbox)
+
+    if not mailboxes:
+        return jsonify(error="At least one mailbox must be selected"), 400
+
+    # hypothesis: user will click on the button in the 600 secs
+    try:
+        alias_suffix = check_suffix_signature(signed_suffix)
+        if not alias_suffix:
+            LOG.w("Alias creation time expired for %s", user)
+            return jsonify(error="Alias creation time is expired, please retry"), 412
+    except Exception:
+        LOG.w("Alias suffix is tampered, user %s", user)
+        return jsonify(error="Tampered suffix"), 400
+
+    if not verify_prefix_suffix(user, alias_prefix, alias_suffix):
+        return jsonify(error="wrong alias prefix or suffix"), 400
+
+    full_alias = alias_prefix + alias_suffix
+    if (
+        Alias.get_by(email=full_alias)
+        or DeletedAlias.get_by(email=full_alias)
+        or DomainDeletedAlias.get_by(email=full_alias)
+    ):
+        LOG.d("full alias already used %s", full_alias)
+        return jsonify(error=f"alias {full_alias} already exists"), 409
+
+    if ".." in full_alias:
+        return (
+            jsonify(error="2 consecutive dot signs aren't allowed in an email address"),
+            400,
+        )
+
+    alias = Alias.create(
+        user_id=user.id,
+        email=full_alias,
+        note=note,
+        name=name or None,
+        mailbox_id=mailboxes[0].id,
+    )
+    Session.flush()
+
+    for i in range(1, len(mailboxes)):
+        AliasMailbox.create(
+            alias_id=alias.id,
+            mailbox_id=mailboxes[i].id,
+        )
+
+    Session.commit()
+
+    if hostname:
+        AliasUsedOn.create(alias_id=alias.id, hostname=hostname, user_id=alias.user_id)
+        Session.commit()
+
+    return (
+        jsonify(alias=full_alias, **serialize_alias_info_v2(get_alias_info_v2(alias))),
+        201,
+    )

app/api/views/new_random_alias.py

@@ -1,17 +1,27 @@
+import tldextract
 from flask import g
 from flask import jsonify, request
-from flask_cors import cross_origin
 
-from app.api.base import api_bp, verify_api_key
-from app.config import MAX_NB_EMAIL_FREE_PLAN
-from app.extensions import db
+from app import parallel_limiter
+from app.alias_suffix import get_alias_suffixes
+from app.api.base import api_bp, require_api_auth
+from app.api.serializer import (
+    get_alias_info_v2,
+    serialize_alias_info_v2,
+)
+from app.config import MAX_NB_EMAIL_FREE_PLAN, ALIAS_LIMIT
+from app.db import Session
+from app.errors import AliasInTrashError
+from app.extensions import limiter
 from app.log import LOG
-from app.models import GenEmail, AliasUsedOn, AliasGeneratorEnum
+from app.models import Alias, AliasUsedOn, AliasGeneratorEnum
+from app.utils import convert_to_id
 
 
 @api_bp.route("/alias/random/new", methods=["POST"])
-@cross_origin()
-@verify_api_key
+@limiter.limit(ALIAS_LIMIT)
+@require_api_auth
+@parallel_limiter.lock(name="alias_creation")
 def new_random_alias():
     """
     Create a new random alias
@@ -37,22 +47,71 @@ def new_random_alias():
     if data:
         note = data.get("note")
 
-    scheme = user.alias_generator
-    mode = request.args.get("mode")
-    if mode:
-        if mode == "word":
-            scheme = AliasGeneratorEnum.word.value
-        elif mode == "uuid":
-            scheme = AliasGeneratorEnum.uuid.value
-        else:
-            return jsonify(error=f"{mode} must be either word or alias"), 400
-
-    gen_email = GenEmail.create_new_random(user=user, scheme=scheme, note=note)
-    db.session.commit()
+    alias = None
 
+    # custom alias suggestion and suffix
     hostname = request.args.get("hostname")
-    if hostname:
-        AliasUsedOn.create(gen_email_id=gen_email.id, hostname=hostname)
-        db.session.commit()
+    if hostname and user.include_website_in_one_click_alias:
+        LOG.d("Use %s to create new alias", hostname)
+        # keep only the domain name of hostname, ignore TLD and subdomain
+        # for ex www.groupon.com -> groupon
+        ext = tldextract.extract(hostname)
+        prefix_suggestion = ext.domain
+        prefix_suggestion = convert_to_id(prefix_suggestion)
 
-    return jsonify(alias=gen_email.email), 201
+        suffixes = get_alias_suffixes(user)
+        # use the first suffix
+        suggested_alias = prefix_suggestion + suffixes[0].suffix
+
+        alias = Alias.get_by(email=suggested_alias)
+
+        # cannot use this alias as it belongs to another user
+        if alias and not alias.user_id == user.id:
+            LOG.d("%s belongs to another user", alias)
+            alias = None
+        elif alias and alias.user_id == user.id:
+            # make sure alias was created for this website
+            if AliasUsedOn.get_by(
+                alias_id=alias.id, hostname=hostname, user_id=alias.user_id
+            ):
+                LOG.d("Use existing alias %s", alias)
+            else:
+                LOG.d("%s wasn't created for this website %s", alias, hostname)
+                alias = None
+        elif not alias:
+            LOG.d("create new alias %s", suggested_alias)
+            try:
+                alias = Alias.create(
+                    user_id=user.id,
+                    email=suggested_alias,
+                    note=note,
+                    mailbox_id=user.default_mailbox_id,
+                    commit=True,
+                )
+            except AliasInTrashError:
+                LOG.i("Alias %s is in trash", suggested_alias)
+                alias = None
+
+    if not alias:
+        scheme = user.alias_generator
+        mode = request.args.get("mode")
+        if mode:
+            if mode == "word":
+                scheme = AliasGeneratorEnum.word.value
+            elif mode == "uuid":
+                scheme = AliasGeneratorEnum.uuid.value
+            else:
+                return jsonify(error=f"{mode} must be either word or uuid"), 400
+
+        alias = Alias.create_new_random(user=user, scheme=scheme, note=note)
+        Session.commit()
+
+    if hostname and not AliasUsedOn.get_by(alias_id=alias.id, hostname=hostname):
+        AliasUsedOn.create(
+            alias_id=alias.id, hostname=hostname, user_id=alias.user_id, commit=True
+        )
+
+    return (
+        jsonify(alias=alias.email, **serialize_alias_info_v2(get_alias_info_v2(alias))),
+        201,
+    )

app/api/views/notification.py

@@ -0,0 +1,83 @@
+from flask import g
+from flask import jsonify
+from flask import request
+
+from app.api.base import api_bp, require_api_auth
+from app.config import PAGE_LIMIT
+from app.db import Session
+from app.models import Notification
+
+
+@api_bp.route("/notifications", methods=["GET"])
+@require_api_auth
+def get_notifications():
+    """
+    Get notifications
+
+    Input:
+    - page: in url. Starts at 0
+
+    Output:
+    - more: boolean. Whether there's more notification to load
+    - notifications: list of notifications.
+        - id
+        - message
+        - title
+        - read
+        - created_at
+    """
+    user = g.user
+    try:
+        page = int(request.args.get("page"))
+    except (ValueError, TypeError):
+        return jsonify(error="page must be provided in request query"), 400
+
+    notifications = (
+        Notification.filter_by(user_id=user.id)
+        .order_by(Notification.read, Notification.created_at.desc())
+        .limit(PAGE_LIMIT + 1)  # load a record more to know whether there's more
+        .offset(page * PAGE_LIMIT)
+        .all()
+    )
+
+    have_more = len(notifications) > PAGE_LIMIT
+
+    return (
+        jsonify(
+            more=have_more,
+            notifications=[
+                {
+                    "id": notification.id,
+                    "message": notification.message,
+                    "title": notification.title,
+                    "read": notification.read,
+                    "created_at": notification.created_at.humanize(),
+                }
+                for notification in notifications[:PAGE_LIMIT]
+            ],
+        ),
+        200,
+    )
+
+
+@api_bp.route("/notifications/<int:notification_id>/read", methods=["POST"])
+@require_api_auth
+def mark_as_read(notification_id):
+    """
+    Mark a notification as read
+    Input:
+        notification_id: in url
+    Output:
+        200 if updated successfully
+
+    """
+    user = g.user
+    notification = Notification.get(notification_id)
+
+    if not notification or notification.user_id != user.id:
+        return jsonify(error="Forbidden"), 403
+
+    notification.read = True
+    Session.commit()
+
+    return jsonify(done=True), 200

app/api/views/phone.py

@@ -0,0 +1,51 @@
+import arrow
+from flask import g
+from flask import jsonify
+
+from app.api.base import api_bp, require_api_auth
+from app.models import (
+    PhoneReservation,
+    PhoneMessage,
+)
+
+
+@api_bp.route("/phone/reservations/<int:reservation_id>", methods=["GET", "POST"])
+@require_api_auth
+def phone_messages(reservation_id):
+    """
+    Return messages during this reservation
+    Output:
+        - messages: list of alias:
+            - id
+            - from_number
+            - body
+            - created_at: e.g. 5 minutes ago
+
+    """
+    user = g.user
+    reservation: PhoneReservation = PhoneReservation.get(reservation_id)
+    if not reservation or reservation.user_id != user.id:
+        return jsonify(error="Invalid reservation"), 400
+
+    phone_number = reservation.number
+    messages = PhoneMessage.filter(
+        PhoneMessage.number_id == phone_number.id,
+        PhoneMessage.created_at > reservation.start,
+        PhoneMessage.created_at < reservation.end,
+    ).all()
+
+    return (
+        jsonify(
+            messages=[
+                {
+                    "id": message.id,
+                    "from_number": message.from_number,
+                    "body": message.body,
+                    "created_at": message.created_at.humanize(),
+                }
+                for message in messages
+            ],
+            ended=reservation.end < arrow.now(),
+        ),
+        200,
+    )

app/api/views/setting.py

@@ -0,0 +1,148 @@
+import arrow
+from flask import jsonify, g, request
+
+from app.api.base import api_bp, require_api_auth
+from app.db import Session
+from app.log import LOG
+from app.models import (
+    User,
+    AliasGeneratorEnum,
+    SLDomain,
+    CustomDomain,
+    SenderFormatEnum,
+    AliasSuffixEnum,
+)
+from app.proton.utils import perform_proton_account_unlink
+
+
+def setting_to_dict(user: User):
+    ret = {
+        "notification": user.notification,
+        "alias_generator": "word"
+        if user.alias_generator == AliasGeneratorEnum.word.value
+        else "uuid",
+        "random_alias_default_domain": user.default_random_alias_domain(),
+        # return the default sender format (AT) in case user uses a non-supported sender format
+        "sender_format": SenderFormatEnum.get_name(user.sender_format)
+        or SenderFormatEnum.AT.name,
+        "random_alias_suffix": AliasSuffixEnum.get_name(user.random_alias_suffix),
+    }
+
+    return ret
+
+
+@api_bp.route("/setting")
+@require_api_auth
+def get_setting():
+    """
+    Return user setting
+    """
+    user = g.user
+
+    return jsonify(setting_to_dict(user))
+
+
+@api_bp.route("/setting", methods=["PATCH"])
+@require_api_auth
+def update_setting():
+    """
+    Update user setting
+    Input:
+    - notification: bool
+    - alias_generator: word|uuid
+    - random_alias_default_domain: str
+    """
+    user = g.user
+    data = request.get_json() or {}
+
+    if "notification" in data:
+        user.notification = data["notification"]
+
+    if "alias_generator" in data:
+        alias_generator = data["alias_generator"]
+        if alias_generator not in ["word", "uuid"]:
+            return jsonify(error="Invalid alias_generator"), 400
+
+        if alias_generator == "word":
+            user.alias_generator = AliasGeneratorEnum.word.value
+        else:
+            user.alias_generator = AliasGeneratorEnum.uuid.value
+
+    if "sender_format" in data:
+        sender_format = data["sender_format"]
+        if not SenderFormatEnum.has_name(sender_format):
+            return jsonify(error="Invalid sender_format"), 400
+
+        user.sender_format = SenderFormatEnum.get_value(sender_format)
+        user.sender_format_updated_at = arrow.now()
+
+    if "random_alias_suffix" in data:
+        random_alias_suffix = data["random_alias_suffix"]
+        if not AliasSuffixEnum.has_name(random_alias_suffix):
+            return jsonify(error="Invalid random_alias_suffix"), 400
+
+        user.random_alias_suffix = AliasSuffixEnum.get_value(random_alias_suffix)
+
+    if "random_alias_default_domain" in data:
+        default_domain = data["random_alias_default_domain"]
+        sl_domain: SLDomain = SLDomain.get_by(domain=default_domain)
+        if sl_domain:
+            if sl_domain.premium_only and not user.is_premium():
+                return jsonify(error="You cannot use this domain"), 400
+
+            user.default_alias_public_domain_id = sl_domain.id
+            user.default_alias_custom_domain_id = None
+        else:
+            custom_domain = CustomDomain.get_by(domain=default_domain)
+            if not custom_domain:
+                return jsonify(error="invalid domain"), 400
+
+            # sanity check
+            if custom_domain.user_id != user.id or not custom_domain.verified:
+                LOG.w("%s cannot use domain %s", user, default_domain)
+                return jsonify(error="invalid domain"), 400
+            else:
+                user.default_alias_custom_domain_id = custom_domain.id
+                user.default_alias_public_domain_id = None
+
+    Session.commit()
+    return jsonify(setting_to_dict(user))
+
+
+@api_bp.route("/setting/domains")
+@require_api_auth
+def get_available_domains_for_random_alias():
+    """
+    Available domains for random alias
+    """
+    user = g.user
+
+    ret = [
+        (is_sl, domain) for is_sl, domain in user.available_domains_for_random_alias()
+    ]
+
+    return jsonify(ret)
+
+
+@api_bp.route("/v2/setting/domains")
+@require_api_auth
+def get_available_domains_for_random_alias_v2():
+    """
+    Available domains for random alias
+    """
+    user = g.user
+
+    ret = [
+        {"domain": domain, "is_custom": not is_sl}
+        for is_sl, domain in user.available_domains_for_random_alias()
+    ]
+
+    return jsonify(ret)
+
+
+@api_bp.route("/setting/unlink_proton_account", methods=["DELETE"])
+@require_api_auth
+def unlink_proton_account():
+    user = g.user
+    perform_proton_account_unlink(user)
+    return jsonify({"ok": True})

app/api/views/sudo.py

@@ -0,0 +1,27 @@
+from flask import jsonify, g, request
+from sqlalchemy_utils.types.arrow import arrow
+
+from app.api.base import api_bp, require_api_auth
+from app.db import Session
+
+
+@api_bp.route("/sudo", methods=["PATCH"])
+@require_api_auth
+def enter_sudo():
+    """
+    Enter sudo mode
+
+    Input
+    - password: user password to validate request to enter sudo mode
+    """
+    user = g.user
+    data = request.get_json() or {}
+    if "password" not in data:
+        return jsonify(error="Invalid password"), 403
+    if not user.check_password(data["password"]):
+        return jsonify(error="Invalid password"), 403
+
+    g.api_key.sudo_mode_at = arrow.now()
+    Session.commit()
+
+    return jsonify(ok=True)

app/api/views/user.py

@@ -0,0 +1,46 @@
+from flask import jsonify, g
+from sqlalchemy_utils.types.arrow import arrow
+
+from app.api.base import api_bp, require_api_sudo, require_api_auth
+from app import config
+from app.extensions import limiter
+from app.log import LOG
+from app.models import Job, ApiToCookieToken
+
+
+@api_bp.route("/user", methods=["DELETE"])
+@require_api_sudo
+def delete_user():
+    """
+    Delete the user. Requires sudo mode.
+
+    """
+    # Schedule delete account job
+    LOG.w("schedule delete account job for %s", g.user)
+    Job.create(
+        name=config.JOB_DELETE_ACCOUNT,
+        payload={"user_id": g.user.id},
+        run_at=arrow.now(),
+        commit=True,
+    )
+    return jsonify(ok=True)
+
+
+@api_bp.route("/user/cookie_token", methods=["GET"])
+@require_api_auth
+@limiter.limit("5/minute")
+def get_api_session_token():
+    """
+    Get a temporary token to exchange it for a cookie based session
+    Output:
+        200 and a temporary random token
+        {
+            token: "asdli3ldq39h9hd3",
+        }
+    """
+    token = ApiToCookieToken.create(
+        user=g.user,
+        api_key_id=g.api_key.id,
+        commit=True,
+    )
+    return jsonify({"token": token.code})

app/api/views/user_info.py

@@ -1,22 +1,161 @@
-from flask import jsonify, request, g
-from flask_cors import cross_origin
-from sqlalchemy import desc
+import base64
+import dataclasses
+from io import BytesIO
+from typing import Optional
 
-from app.api.base import api_bp, verify_api_key
-from app.config import EMAIL_DOMAIN
-from app.extensions import db
-from app.log import LOG
-from app.models import AliasUsedOn, GenEmail, User
-from app.utils import convert_to_id, random_word
+from flask import jsonify, g, request, make_response
+
+from app import s3, config
+from app.api.base import api_bp, require_api_auth
+from app.config import SESSION_COOKIE_NAME
+from app.dashboard.views.index import get_stats
+from app.db import Session
+from app.models import ApiKey, File, PartnerUser, User
+from app.proton.utils import get_proton_partner
+from app.session import logout_session
+from app.utils import random_string
+
+
+def get_connected_proton_address(user: User) -> Optional[str]:
+    proton_partner = get_proton_partner()
+    partner_user = PartnerUser.get_by(user_id=user.id, partner_id=proton_partner.id)
+    if partner_user is None:
+        return None
+    return partner_user.partner_email
+
+
+def user_to_dict(user: User) -> dict:
+    ret = {
+        "name": user.name or "",
+        "is_premium": user.is_premium(),
+        "email": user.email,
+        "in_trial": user.in_trial(),
+        "max_alias_free_plan": user.max_alias_for_free_account(),
+        "connected_proton_address": None,
+        "can_create_reverse_alias": user.can_create_contacts(),
+    }
+
+    if config.CONNECT_WITH_PROTON:
+        ret["connected_proton_address"] = get_connected_proton_address(user)
+
+    if user.profile_picture_id:
+        ret["profile_picture_url"] = user.profile_picture.get_url()
+    else:
+        ret["profile_picture_url"] = None
+
+    return ret
 
 
 @api_bp.route("/user_info")
-@cross_origin()
-@verify_api_key
+@require_api_auth
 def user_info():
     """
     Return user info given the api-key
+
+    Output as json
+    - name
+    - is_premium
+    - email
+    - in_trial
+    - max_alias_free
+    - is_connected_with_proton
+    - can_create_reverse_alias
     """
     user = g.user
 
-    return jsonify({"name": user.name, "is_premium": user.is_premium()})
+    return jsonify(user_to_dict(user))
+
+
+@api_bp.route("/user_info", methods=["PATCH"])
+@require_api_auth
+def update_user_info():
+    """
+    Input
+    - profile_picture (optional): base64 of the profile picture. Set to null to remove the profile picture
+    - name (optional)
+    """
+    user = g.user
+    data = request.get_json() or {}
+
+    if "profile_picture" in data:
+        if data["profile_picture"] is None:
+            if user.profile_picture_id:
+                file = user.profile_picture
+                user.profile_picture_id = None
+                Session.flush()
+                if file:
+                    File.delete(file.id)
+                    s3.delete(file.path)
+                    Session.flush()
+        else:
+            raw_data = base64.decodebytes(data["profile_picture"].encode())
+            file_path = random_string(30)
+            file = File.create(user_id=user.id, path=file_path)
+            Session.flush()
+            s3.upload_from_bytesio(file_path, BytesIO(raw_data))
+            user.profile_picture_id = file.id
+            Session.flush()
+
+    if "name" in data:
+        user.name = data["name"]
+
+    Session.commit()
+
+    return jsonify(user_to_dict(user))
+
+
+@api_bp.route("/api_key", methods=["POST"])
+@require_api_auth
+def create_api_key():
+    """Used to create a new api key
+    Input:
+    - device
+
+    Output:
+    - api_key
+    """
+    data = request.get_json()
+    if not data:
+        return jsonify(error="request body cannot be empty"), 400
+
+    device = data.get("device")
+
+    api_key = ApiKey.create(user_id=g.user.id, name=device)
+    Session.commit()
+
+    return jsonify(api_key=api_key.code), 201
+
+
+@api_bp.route("/logout", methods=["GET"])
+@require_api_auth
+def logout():
+    """
+    Log user out on the web, i.e. remove the cookie
+
+    Output:
+    - 200
+    """
+    logout_session()
+    response = make_response(jsonify(msg="User is logged out"), 200)
+    response.delete_cookie(SESSION_COOKIE_NAME)
+
+    return response
+
+
+@api_bp.route("/stats")
+@require_api_auth
+def user_stats():
+    """
+    Return stats
+
+    Output as json
+    - nb_alias
+    - nb_forward
+    - nb_reply
+    - nb_block
+
+    """
+    user = g.user
+    stats = get_stats(user)
+
+    return jsonify(dataclasses.asdict(stats))

app/auth/__init__.py

@@ -9,6 +9,33 @@ from .views import (
     github,
     google,
     facebook,
+    proton,
     change_email,
     mfa,
+    fido,
+    social,
+    recovery,
+    api_to_cookie,
+    oidc,
 )
+
+__all__ = [
+    "login",
+    "logout",
+    "register",
+    "activate",
+    "resend_activation",
+    "reset_password",
+    "forgot_password",
+    "github",
+    "google",
+    "facebook",
+    "proton",
+    "change_email",
+    "mfa",
+    "fido",
+    "social",
+    "recovery",
+    "api_to_cookie",
+    "oidc",
+]

app/auth/templates/auth/change_email.html

@@ -1,30 +0,0 @@
-{% extends "single.html" %}
-
-{% block title %}
-  Change Email
-{% endblock %}
-
-{% block single_content %}
-  {% if error %}
-    <div class="text-danger text-center mb-4">{{ error }}</div>
-  {% endif %}
-
-  {% if incorrect_code %}
-    <div class="text-danger text-center h4">
-      The link is incorrect. <br><br>
-      Please go to <a href="{{ url_for('dashboard.setting') }}"
-                      class="btn btn-warning">settings</a>
-      page to re-send confirmation email.
-    </div>
-  {% endif %}
-
-  {% if expired_code %}
-    <div class="text-danger text-center h4">
-      The link is already expired. <br><br>
-      Please go to <a href="{{ url_for('dashboard.setting') }}"
-                      class="btn btn-warning">settings</a>
-      page to re-send confirmation email.
-    </div>
-  {% endif %}
-
-{% endblock %}

app/auth/templates/auth/login.html

@@ -1,102 +0,0 @@
-{% extends "single.html" %}
-
-
-{% block title %}
-  Login
-{% endblock %}
-
-
-{% block head %}
-  <style>
-    .col-login {
-      max-width: 48rem;
-    }
-  </style>
-{% endblock %}
-
-{% block single_content %}
-  <h1 class="h2 text-center">Welcome back!</h1>
-
-  <div class="row">
-    <div class="col-md-6">
-      {% if show_resend_activation %}
-        <div class="text-center text-muted small mb-4">
-          You haven't received the activation email?
-          <a href="{{ url_for('auth.resend_activation') }}">Resend</a>
-        </div>
-      {% endif %}
-
-      <div class="card">
-        <form method="post">
-          {{ form.csrf_token }}
-          <div class="card-body p-6">
-            <div class="form-group">
-              <label class="form-label">Email address</label>
-              {{ form.email(class="form-control", type="email") }}
-              {{ render_field_errors(form.email) }}
-            </div>
-
-            <div class="form-group">
-              <label class="form-label">
-                Password
-              </label>
-              {{ form.password(class="form-control", type="password") }}
-              {{ render_field_errors(form.password) }}
-              <div class="text-muted">
-                <a href="{{ url_for('auth.forgot_password') }}" class="small">
-                  I forgot my password
-                </a>
-              </div>
-            </div>
-
-            <div class="form-footer">
-              <button type="submit" class="btn btn-primary btn-block">Log in</button>
-            </div>
-
-            <div class="text-center text-muted mt-2">
-              Don't have an account yet? <a href="{{ url_for('auth.register') }}">Sign up</a>
-            </div>
-
-          </div>
-        </form>
-      </div>
-    </div>
-
-    <div class="col-md-6">
-      <div class="card">
-        <div class="card-body p-6">
-          <div class="card-title text-center">Or with social login
-          </div>
-
-          <a href="{{ url_for('auth.github_login', next=next_url) }}"
-             class="btn btn-block btn-social btn-github">
-            <i class="fa fa-github"></i> Sign in with Github
-          </a>
-
-          <a href="{{ url_for('auth.google_login', next=next_url) }}"
-             class="btn btn-block btn-social btn-google">
-            <i class="fa fa-google"></i> Sign in with Google
-          </a>
-
-          <a href="{{ url_for('auth.facebook_login', next=next_url) }}"
-             class="btn btn-block btn-social btn-facebook">
-            <i class="fa fa-facebook"></i> Sign in with Facebook
-          </a>
-        </div>
-        <div class="text-center p-3" style="font-size: 12px; font-weight: 300; margin: auto">
-
-          We do not use the Facebook/Google SDK to avoid their trackers. <br>
-
-          However when using a social login button, please keep in mind that this social network will <b>know</b> that
-          you are using SimpleLogin.
-
-          <span class="badge badge-warning">Warning</span>
-
-        </div>
-      </div>
-
-
-    </div>
-  </div>
-
-{% endblock %}

app/auth/templates/auth/logout.html

@@ -1,14 +0,0 @@
-{% extends "single.html" %}
-
-{% block title %}
-  Logout
-{% endblock %}
-
-{% block single_content %}
-  <div class="text-center text-muted">
-    You are logged out.
-
-    <a href="{{ url_for('auth.login') }}">Login</a>
-  </div>
-
-{% endblock %}

app/auth/templates/auth/mfa.html

@@ -1,33 +0,0 @@
-{% extends "single.html" %}
-
-
-{% block title %}
-  MFA
-{% endblock %}
-
-
-{% block single_content %}
-  <div class="bg-white p-6" style="margin: auto">
-
-    <div>
-      Your account is protected with multi-factor authentication (MFA). <br>
-      To continue with the sign-in you need to provide the access code from your authenticator.
-    </div>
-
-    <form method="post">
-      {{ otp_token_form.csrf_token }}
-      <input type="hidden" name="form-name" value="create">
-
-      <div class="font-weight-bold mt-5">Token</div>
-      <div class="small-text">Please enter the 6-digit number displayed in your MFA application (Google Authenticator,
-        Authy) here
-      </div>
-
-      {{ otp_token_form.token(class="form-control", autofocus="true") }}
-      {{ render_field_errors(otp_token_form.token) }}
-      <button class="btn btn-success mt-2">Validate</button>
-    </form>
-
-  </div>
-
-{% endblock %}

app/auth/templates/auth/register.html

@@ -1,96 +0,0 @@
-{% extends "single.html" %}
-
-{% block title %}
-  Register
-{% endblock %}
-
-{% block head %}
-  <style>
-    .col-login {
-      max-width: 48rem;
-    }
-  </style>
-{% endblock %}
-
-
-{% block single_content %}
-  <h1 class="h3 text-center">Create your SimpleLogin account now</h1>
-
-  <div class="row">
-    <div class="col-md-6">
-      <div class="card">
-        <form method="post">
-          {{ form.csrf_token }}
-          <div class="card-body p-6">
-            <div class="form-group">
-              <label class="form-label">Email address</label>
-              {{ form.email(class="form-control", type="email") }}
-              {{ render_field_errors(form.email) }}
-            </div>
-
-            <div class="form-group">
-              <label class="form-label">Password</label>
-              {{ form.password(class="form-control", type="password") }}
-              {{ render_field_errors(form.password) }}
-            </div>
-
-            <!-- TODO: add terms
-            <div class="form-group">
-              <label class="custom-control custom-checkbox">
-                <input type="checkbox" class="custom-control-input"/>
-                <span class="custom-control-label">Agree the <a href="terms.html">terms and policy</a></span>
-              </label>
-            </div>
-            -->
-
-            <div class="form-footer">
-              <button type="submit" class="btn btn-primary btn-block">Create your SimpleLogin account</button>
-            </div>
-          </div>
-        </form>
-
-        <div class="text-center text-muted mb-6">
-          Already have account? <a href="{{ url_for('auth.login') }}">Sign in</a>
-        </div>
-      </div>
-
-
-    </div>
-
-    <div class="col-md-6">
-      <div class="card">
-        <div class="card-body p-6">
-          <div class="card-title text-center">Or with social login</div>
-
-          <a href="{{ url_for('auth.github_login', next=next_url) }}"
-             class="btn btn-block btn-social btn-github">
-            <i class="fa fa-github"></i> Sign up with Github
-          </a>
-
-          <a href="{{ url_for('auth.google_login', next=next_url) }}"
-             class="btn btn-block btn-social btn-google">
-            <i class="fa fa-google"></i> Sign up with Google
-          </a>
-
-          <a href="{{ url_for('auth.facebook_login', next=next_url) }}"
-             class="btn btn-block btn-social btn-facebook">
-            <i class="fa fa-facebook"></i> Sign up with Facebook
-          </a>
-
-        </div>
-        <div class="text-center p-3" style="font-size: 12px; font-weight: 300; margin: auto">
-
-          We do not use the Facebook/Google SDK to avoid their trackers. <br>
-
-          However when using a social login button, please keep in mind that this social network will <b>know</b> that
-          you are using SimpleLogin.
-
-          <span class="badge badge-warning">Warning</span>
-
-        </div>
-
-      </div>
-    </div>
-  </div>
-
-{% endblock %}

app/auth/templates/auth/register_waiting_activation.html

@@ -1,22 +0,0 @@
-{% extends "single.html" %}
-
-{% block title %}
-  Activation Email Sent
-{% endblock %}
-
-{% block single_content %}
-  <div class="text-center bg-white p-5" style="max-width: 50rem">
-    <h1>
-      An email to validate your email is on its way.
-    </h1>
-
-    <h3>
-      Please check your inbox/spam folder.
-    </h3>
-    <small>
-      Yeah we know. An email to confirm an email ...
-    </small>
-    </h1>
-  </div>
-
-{% endblock %}

app/auth/views/activate.py

@@ -1,14 +1,19 @@
-from flask import request, redirect, url_for, flash, render_template
+from flask import request, redirect, url_for, flash, render_template, g
 from flask_login import login_user, current_user
 
 from app import email_utils
 from app.auth.base import auth_bp
-from app.extensions import db
+from app.db import Session
+from app.extensions import limiter
 from app.log import LOG
 from app.models import ActivationCode
+from app.utils import sanitize_next_url
 
 
 @auth_bp.route("/activate", methods=["GET", "POST"])
+@limiter.limit(
+    "10/minute", deduct_when=lambda r: hasattr(g, "deduct_limit") and g.deduct_limit
+)
 def activate():
     if current_user.is_authenticated:
         return (
@@ -21,6 +26,8 @@ def activate():
     activation_code: ActivationCode = ActivationCode.get_by(code=code)
 
     if not activation_code:
+        # Trigger rate limiter
+        g.deduct_limit = True
         return (
             render_template(
                 "auth/activate.html", error="Activation code cannot be found"
@@ -41,19 +48,22 @@ def activate():
     user = activation_code.user
     user.activated = True
     login_user(user)
-    email_utils.send_welcome_email(user)
 
     # activation code is to be used only once
     ActivationCode.delete(activation_code.id)
-    db.session.commit()
+    Session.commit()
 
     flash("Your account has been activated", "success")
 
+    email_utils.send_welcome_email(user)
+
     # The activation link contains the original page, for ex authorize page
     if "next" in request.args:
-        next_url = request.args.get("next")
-        LOG.debug("redirect user to %s", next_url)
+        next_url = sanitize_next_url(request.args.get("next"))
+        LOG.d("redirect user to %s", next_url)
         return redirect(next_url)
     else:
-        LOG.debug("redirect user to dashboard")
+        LOG.d("redirect user to dashboard")
         return redirect(url_for("dashboard.index"))
+        # todo: redirect to account_activated page when more features are added into the browser extension
+        # return redirect(url_for("onboarding.account_activated"))

app/auth/views/api_to_cookie.py

@@ -0,0 +1,30 @@
+import arrow
+from flask import redirect, url_for, request, flash
+from flask_login import login_user
+
+from app.auth.base import auth_bp
+from app.models import ApiToCookieToken
+from app.utils import sanitize_next_url
+
+
+@auth_bp.route("/api_to_cookie", methods=["GET"])
+def api_to_cookie():
+    code = request.args.get("token")
+    if not code:
+        flash("Missing token", "error")
+        return redirect(url_for("auth.login"))
+
+    token = ApiToCookieToken.get_by(code=code)
+    if not token or token.created_at < arrow.now().shift(minutes=-5):
+        flash("Missing token", "error")
+        return redirect(url_for("auth.login"))
+
+    user = token.user
+    ApiToCookieToken.delete(token.id, commit=True)
+    login_user(user)
+
+    next_url = sanitize_next_url(request.args.get("next"))
+    if next_url:
+        return redirect(next_url)
+    else:
+        return redirect(url_for("dashboard.index"))

app/auth/views/change_email.py

@@ -2,28 +2,37 @@ from flask import request, flash, render_template, redirect, url_for
 from flask_login import login_user
 
 from app.auth.base import auth_bp
-from app.extensions import db
-from app.models import EmailChange
+from app.db import Session
+from app.extensions import limiter
+from app.log import LOG
+from app.models import EmailChange, ResetPasswordCode
 
 
 @auth_bp.route("/change_email", methods=["GET", "POST"])
+@limiter.limit("3/hour")
 def change_email():
     code = request.args.get("code")
 
     email_change: EmailChange = EmailChange.get_by(code=code)
 
     if not email_change:
-        return render_template("auth/change_email.html", incorrect_code=True)
+        return render_template("auth/change_email.html")
 
     if email_change.is_expired():
-        return render_template("auth/change_email.html", expired_code=True)
+        # delete the expired email
+        EmailChange.delete(email_change.id)
+        Session.commit()
+        return render_template("auth/change_email.html")
 
     user = email_change.user
+    old_email = user.email
     user.email = email_change.new_email
 
     EmailChange.delete(email_change.id)
-    db.session.commit()
+    ResetPasswordCode.filter_by(user_id=user.id).delete()
+    Session.commit()
 
+    LOG.i(f"User {user} has changed their email from {old_email} to {user.email}")
     flash("Your new email has been updated", "success")
 
     login_user(user)

app/auth/views/facebook.py

@@ -1,22 +1,19 @@
 from flask import request, session, redirect, url_for, flash
-from flask_login import login_user
 from requests_oauthlib import OAuth2Session
 from requests_oauthlib.compliance_fixes import facebook_compliance_fix
 
-from app import email_utils
 from app.auth.base import auth_bp
 from app.auth.views.google import create_file_from_url
 from app.config import (
     URL,
     FACEBOOK_CLIENT_ID,
     FACEBOOK_CLIENT_SECRET,
-    DISABLE_REGISTRATION,
 )
-from app.extensions import db
+from app.db import Session
 from app.log import LOG
 from app.models import User, SocialAuth
 from .login_utils import after_login
-from ...email_utils import can_be_used_as_personal_email, email_already_used
+from ...utils import sanitize_email, sanitize_next_url
 
 _authorization_base_url = "https://www.facebook.com/dialog/oauth"
 _token_url = "https://graph.facebook.com/oauth/access_token"
@@ -33,7 +30,7 @@ def facebook_login():
     # to avoid flask-login displaying the login error message
     session.pop("_flashes", None)
 
-    next_url = request.args.get("next")
+    next_url = sanitize_next_url(request.args.get("next"))
 
     # Facebook does not allow to append param to redirect_uri
     # we need to pass the next url by session
@@ -65,7 +62,7 @@ def facebook_callback():
         redirect_uri=_redirect_uri,
     )
     facebook = facebook_compliance_fix(facebook)
-    token = facebook.fetch_token(
+    facebook.fetch_token(
         _token_url,
         client_secret=FACEBOOK_CLIENT_SECRET,
         authorization_response=request.url,
@@ -95,6 +92,7 @@ def facebook_callback():
         )
         return redirect(url_for("auth.register"))
 
+    email = sanitize_email(email)
     user = User.get_by(email=email)
 
     picture_url = facebook_user_data.get("picture", {}).get("data", {}).get("url")
@@ -102,47 +100,28 @@ def facebook_callback():
     if user:
         if picture_url and not user.profile_picture_id:
             LOG.d("set user profile picture to %s", picture_url)
-            file = create_file_from_url(picture_url)
+            file = create_file_from_url(user, picture_url)
             user.profile_picture_id = file.id
-            db.session.commit()
+            Session.commit()
 
-    # create user
     else:
-        if DISABLE_REGISTRATION:
-            flash("Registration is closed", "error")
-            return redirect(url_for("auth.login"))
-
-        if not can_be_used_as_personal_email(email) or email_already_used(email):
-            flash(f"You cannot use {email} as your personal inbox.", "error")
-            return redirect(url_for("auth.login"))
-
-        LOG.d("create facebook user with %s", facebook_user_data)
-        user = User.create(
-            email=email.lower(), name=facebook_user_data["name"], activated=True
+        flash(
+            "Sorry you cannot sign up via Facebook, please use email/password sign-up instead",
+            "error",
         )
-
-        if picture_url:
-            LOG.d("set user profile picture to %s", picture_url)
-            file = create_file_from_url(picture_url)
-            user.profile_picture_id = file.id
-
-        db.session.commit()
-        login_user(user)
-        email_utils.send_welcome_email(user)
-
-        flash(f"Welcome to SimpleLogin {user.name}!", "success")
+        return redirect(url_for("auth.register"))
 
     next_url = None
     # The activation link contains the original page, for ex authorize page
     if "facebook_next_url" in session:
         next_url = session["facebook_next_url"]
-        LOG.debug("redirect user to %s", next_url)
+        LOG.d("redirect user to %s", next_url)
 
         # reset the next_url to avoid user getting redirected at each login :)
         session.pop("facebook_next_url", None)
 
     if not SocialAuth.get_by(user_id=user.id, social="facebook"):
         SocialAuth.create(user_id=user.id, social="facebook")
-        db.session.commit()
+        Session.commit()
 
     return after_login(user, next_url)

app/auth/views/fido.py

@@ -0,0 +1,173 @@
+import json
+import secrets
+from time import time
+
+import webauthn
+from flask import (
+    request,
+    render_template,
+    redirect,
+    url_for,
+    flash,
+    session,
+    make_response,
+    g,
+)
+from flask_login import login_user
+from flask_wtf import FlaskForm
+from wtforms import HiddenField, validators, BooleanField
+
+from app.auth.base import auth_bp
+from app.config import MFA_USER_ID
+from app.config import RP_ID, URL
+from app.db import Session
+from app.extensions import limiter
+from app.log import LOG
+from app.models import User, Fido, MfaBrowser
+from app.utils import sanitize_next_url
+
+
+class FidoTokenForm(FlaskForm):
+    sk_assertion = HiddenField("sk_assertion", validators=[validators.DataRequired()])
+    remember = BooleanField(
+        "attr", default=False, description="Remember this browser for 30 days"
+    )
+
+
+@auth_bp.route("/fido", methods=["GET", "POST"])
+@limiter.limit(
+    "10/minute", deduct_when=lambda r: hasattr(g, "deduct_limit") and g.deduct_limit
+)
+def fido():
+    # passed from login page
+    user_id = session.get(MFA_USER_ID)
+
+    # user access this page directly without passing by login page
+    if not user_id:
+        flash("Unknown error, redirect back to main page", "warning")
+        return redirect(url_for("auth.login"))
+
+    user = User.get(user_id)
+
+    if not (user and user.fido_enabled()):
+        flash("Only user with security key linked should go to this page", "warning")
+        return redirect(url_for("auth.login"))
+
+    auto_activate = True
+    fido_token_form = FidoTokenForm()
+
+    next_url = sanitize_next_url(request.args.get("next"))
+
+    if request.cookies.get("mfa"):
+        browser = MfaBrowser.get_by(token=request.cookies.get("mfa"))
+        if browser and not browser.is_expired() and browser.user_id == user.id:
+            login_user(user)
+            flash("Welcome back!", "success")
+            # Redirect user to correct page
+            return redirect(next_url or url_for("dashboard.index"))
+        else:
+            # Trigger rate limiter
+            g.deduct_limit = True
+
+    # Handling POST requests
+    if fido_token_form.validate_on_submit():
+        try:
+            sk_assertion = json.loads(fido_token_form.sk_assertion.data)
+        except Exception:
+            flash("Key verification failed. Error: Invalid Payload", "warning")
+            return redirect(url_for("auth.login"))
+
+        challenge = session["fido_challenge"]
+
+        try:
+            fido_key = Fido.get_by(
+                uuid=user.fido_uuid, credential_id=sk_assertion["id"]
+            )
+            webauthn_user = webauthn.WebAuthnUser(
+                user.fido_uuid,
+                user.email,
+                user.name if user.name else user.email,
+                False,
+                fido_key.credential_id,
+                fido_key.public_key,
+                fido_key.sign_count,
+                RP_ID,
+            )
+            webauthn_assertion_response = webauthn.WebAuthnAssertionResponse(
+                webauthn_user, sk_assertion, challenge, URL, uv_required=False
+            )
+            new_sign_count = webauthn_assertion_response.verify()
+        except Exception as e:
+            LOG.w(f"An error occurred in WebAuthn verification process: {e}")
+            flash("Key verification failed.", "warning")
+            # Trigger rate limiter
+            g.deduct_limit = True
+            auto_activate = False
+        else:
+            user.fido_sign_count = new_sign_count
+            Session.commit()
+            del session[MFA_USER_ID]
+
+            session["sudo_time"] = int(time())
+            login_user(user)
+            flash("Welcome back!", "success")
+
+            # Redirect user to correct page
+            response = make_response(redirect(next_url or url_for("dashboard.index")))
+
+            if fido_token_form.remember.data:
+                browser = MfaBrowser.create_new(user=user)
+                Session.commit()
+                response.set_cookie(
+                    "mfa",
+                    value=browser.token,
+                    expires=browser.expires.datetime,
+                    secure=True if URL.startswith("https") else False,
+                    httponly=True,
+                    samesite="Lax",
+                )
+
+            return response
+
+    # Prepare information for key registration process
+    session.pop("challenge", None)
+    challenge = secrets.token_urlsafe(32)
+
+    session["fido_challenge"] = challenge.rstrip("=")
+
+    fidos = Fido.filter_by(uuid=user.fido_uuid).all()
+    webauthn_users = []
+    for fido in fidos:
+        webauthn_users.append(
+            webauthn.WebAuthnUser(
+                user.fido_uuid,
+                user.email,
+                user.name if user.name else user.email,
+                False,
+                fido.credential_id,
+                fido.public_key,
+                fido.sign_count,
+                RP_ID,
+            )
+        )
+
+    webauthn_assertion_options = webauthn.WebAuthnAssertionOptions(
+        webauthn_users, challenge
+    )
+    webauthn_assertion_options = webauthn_assertion_options.assertion_dict
+    try:
+        # HACK: We need to upgrade to webauthn > 1 so it can support specifying the transports
+        for credential in webauthn_assertion_options["allowCredentials"]:
+            del credential["transports"]
+    except KeyError:
+        # Should never happen but...
+        pass
+
+    return render_template(
+        "auth/fido.html",
+        fido_token_form=fido_token_form,
+        webauthn_assertion_options=webauthn_assertion_options,
+        enable_otp=user.enable_otp,
+        auto_activate=auto_activate,
+        next_url=next_url,
+    )

app/auth/views/forgot_password.py

@@ -1,10 +1,13 @@
-from flask import request, render_template, redirect, url_for
+from flask import request, render_template, flash, g
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators
 
 from app.auth.base import auth_bp
-from app.dashboard.views.setting import send_reset_password_email
+from app.dashboard.views.account_setting import send_reset_password_email
+from app.extensions import limiter
+from app.log import LOG
 from app.models import User
+from app.utils import sanitize_email, canonicalize_email
 
 
 class ForgotPasswordForm(FlaskForm):
@@ -12,19 +15,27 @@ class ForgotPasswordForm(FlaskForm):
 
 
 @auth_bp.route("/forgot_password", methods=["GET", "POST"])
+@limiter.limit(
+    "10/hour", deduct_when=lambda r: hasattr(g, "deduct_limit") and g.deduct_limit
+)
 def forgot_password():
     form = ForgotPasswordForm(request.form)
 
     if form.validate_on_submit():
-        email = form.email.data
+        # Trigger rate limiter
+        g.deduct_limit = True
 
-        user = User.get_by(email=email)
+        flash(
+            "If your email is correct, you are going to receive an email to reset your password",
+            "success",
+        )
 
-        if not user:
-            error = "No such user, are you sure the email is correct?"
-            return render_template("auth/forgot_password.html", form=form, error=error)
+        email = sanitize_email(form.email.data)
+        canonical_email = canonicalize_email(email)
+        user = User.get_by(email=email) or User.get_by(email=canonical_email)
 
-        send_reset_password_email(user)
-        return redirect(url_for("auth.forgot_password"))
+        if user:
+            LOG.d("Send forgot password email to %s", user)
+            send_reset_password_email(user)
 
     return render_template("auth/forgot_password.html", form=form)

app/auth/views/github.py

@@ -1,16 +1,13 @@
 from flask import request, session, redirect, flash, url_for
-from flask_login import login_user
 from requests_oauthlib import OAuth2Session
 
-from app import email_utils
 from app.auth.base import auth_bp
 from app.auth.views.login_utils import after_login
-from app.config import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, URL, DISABLE_REGISTRATION
-from app.email_utils import can_be_used_as_personal_email, email_already_used
-from app.extensions import db
+from app.config import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, URL
+from app.db import Session
 from app.log import LOG
 from app.models import User, SocialAuth
-from app.utils import encode_url
+from app.utils import encode_url, sanitize_email, sanitize_next_url
 
 _authorization_base_url = "https://github.com/login/oauth/authorize"
 _token_url = "https://github.com/login/oauth/access_token"
@@ -22,7 +19,7 @@ _redirect_uri = URL + "/auth/github/callback"
 
 @auth_bp.route("/github/login")
 def github_login():
-    next_url = request.args.get("next")
+    next_url = sanitize_next_url(request.args.get("next"))
     if next_url:
         redirect_uri = _redirect_uri + "?next=" + encode_url(next_url)
     else:
@@ -51,7 +48,7 @@ def github_callback():
         scope=["user:email"],
         redirect_uri=_redirect_uri,
     )
-    token = github.fetch_token(
+    github.fetch_token(
         _token_url,
         client_secret=GITHUB_CLIENT_SECRET,
         authorization_response=request.url,
@@ -78,41 +75,28 @@ def github_callback():
             break
 
     if not email:
-        LOG.error(f"cannot get email for github user {github_user_data} {emails}")
+        LOG.e(f"cannot get email for github user {github_user_data} {emails}")
         flash(
             "Cannot get a valid email from Github, please another way to login/sign up",
             "error",
         )
         return redirect(url_for("auth.login"))
 
-    email = email.lower()
+    email = sanitize_email(email)
     user = User.get_by(email=email)
 
-    # create user
     if not user:
-        if DISABLE_REGISTRATION:
-            flash("Registration is closed", "error")
-            return redirect(url_for("auth.login"))
-
-        if not can_be_used_as_personal_email(email) or email_already_used(email):
-            flash(f"You cannot use {email} as your personal inbox.", "error")
-            return redirect(url_for("auth.login"))
-
-        LOG.d("create github user")
-        user = User.create(
-            email=email.lower(), name=github_user_data.get("name") or "", activated=True
+        flash(
+            "Sorry you cannot sign up via Github, please use email/password sign-up instead",
+            "error",
         )
-        db.session.commit()
-        login_user(user)
-        email_utils.send_welcome_email(user)
-
-        flash(f"Welcome to SimpleLogin {user.name}!", "success")
+        return redirect(url_for("auth.register"))
 
     if not SocialAuth.get_by(user_id=user.id, social="github"):
         SocialAuth.create(user_id=user.id, social="github")
-        db.session.commit()
+        Session.commit()
 
     # The activation link contains the original page, for ex authorize page
-    next_url = request.args.get("next") if request.args else None
+    next_url = sanitize_next_url(request.args.get("next")) if request.args else None
 
     return after_login(user, next_url)

app/auth/views/google.py

@@ -1,16 +1,14 @@
 from flask import request, session, redirect, flash, url_for
-from flask_login import login_user
 from requests_oauthlib import OAuth2Session
 
-from app import s3, email_utils
+from app import s3
 from app.auth.base import auth_bp
-from app.config import URL, GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, DISABLE_REGISTRATION
-from app.extensions import db
+from app.config import URL, GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET
+from app.db import Session
 from app.log import LOG
 from app.models import User, File, SocialAuth
-from app.utils import random_string
+from app.utils import random_string, sanitize_email, sanitize_next_url
 from .login_utils import after_login
-from ...email_utils import can_be_used_as_personal_email, email_already_used
 
 _authorization_base_url = "https://accounts.google.com/o/oauth2/v2/auth"
 _token_url = "https://www.googleapis.com/oauth2/v4/token"
@@ -31,7 +29,7 @@ def google_login():
     # to avoid flask-login displaying the login error message
     session.pop("_flashes", None)
 
-    next_url = request.args.get("next")
+    next_url = sanitize_next_url(request.args.get("next"))
 
     # Google does not allow to append param to redirect_url
     # we need to pass the next url by session
@@ -55,11 +53,12 @@ def google_callback():
 
     google = OAuth2Session(
         GOOGLE_CLIENT_ID,
-        state=session["oauth_state"],
+        # some how Google Login fails with oauth_state KeyError
+        # state=session["oauth_state"],
         scope=_scope,
         redirect_uri=_redirect_uri,
     )
-    token = google.fetch_token(
+    google.fetch_token(
         _token_url,
         client_secret=GOOGLE_CLIENT_SECRET,
         authorization_response=request.url,
@@ -80,7 +79,7 @@ def google_callback():
         "https://www.googleapis.com/oauth2/v1/userinfo"
     ).json()
 
-    email = google_user_data["email"]
+    email = sanitize_email(google_user_data["email"])
     user = User.get_by(email=email)
 
     picture_url = google_user_data.get("picture")
@@ -88,58 +87,39 @@ def google_callback():
     if user:
         if picture_url and not user.profile_picture_id:
             LOG.d("set user profile picture to %s", picture_url)
-            file = create_file_from_url(picture_url)
+            file = create_file_from_url(user, picture_url)
             user.profile_picture_id = file.id
-            db.session.commit()
-    # create user
+            Session.commit()
     else:
-        if DISABLE_REGISTRATION:
-            flash("Registration is closed", "error")
-            return redirect(url_for("auth.login"))
-
-        if not can_be_used_as_personal_email(email) or email_already_used(email):
-            flash(f"You cannot use {email} as your personal inbox.", "error")
-            return redirect(url_for("auth.login"))
-
-        LOG.d("create google user with %s", google_user_data)
-        user = User.create(
-            email=email.lower(), name=google_user_data["name"], activated=True
+        flash(
+            "Sorry you cannot sign up via Google, please use email/password sign-up instead",
+            "error",
         )
-
-        if picture_url:
-            LOG.d("set user profile picture to %s", picture_url)
-            file = create_file_from_url(picture_url)
-            user.profile_picture_id = file.id
-
-        db.session.commit()
-        login_user(user)
-        email_utils.send_welcome_email(user)
-
-        flash(f"Welcome to SimpleLogin {user.name}!", "success")
+        return redirect(url_for("auth.register"))
 
     next_url = None
     # The activation link contains the original page, for ex authorize page
     if "google_next_url" in session:
         next_url = session["google_next_url"]
-        LOG.debug("redirect user to %s", next_url)
+        LOG.d("redirect user to %s", next_url)
 
         # reset the next_url to avoid user getting redirected at each login :)
         session.pop("google_next_url", None)
 
     if not SocialAuth.get_by(user_id=user.id, social="google"):
         SocialAuth.create(user_id=user.id, social="google")
-        db.session.commit()
+        Session.commit()
 
     return after_login(user, next_url)
 
 
-def create_file_from_url(url) -> File:
+def create_file_from_url(user, url) -> File:
     file_path = random_string(30)
-    file = File.create(path=file_path)
+    file = File.create(path=file_path, user_id=user.id)
 
     s3.upload_from_url(url, file_path)
 
-    db.session.flush()
+    Session.flush()
     LOG.d("upload file %s to s3", file)
 
     return file

app/auth/views/login.py

@@ -1,12 +1,16 @@
-from flask import request, render_template, redirect, url_for, flash
+from flask import request, render_template, redirect, url_for, flash, g
 from flask_login import current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators
 
 from app.auth.base import auth_bp
 from app.auth.views.login_utils import after_login
+from app.config import CONNECT_WITH_PROTON, CONNECT_WITH_OIDC_ICON, OIDC_CLIENT_ID
+from app.events.auth_event import LoginEvent
+from app.extensions import limiter
 from app.log import LOG
 from app.models import User
+from app.utils import sanitize_email, sanitize_next_url, canonicalize_email
 
 
 class LoginForm(FlaskForm):
@@ -15,29 +19,56 @@ class LoginForm(FlaskForm):
 
 
 @auth_bp.route("/login", methods=["GET", "POST"])
+@limiter.limit(
+    "10/minute", deduct_when=lambda r: hasattr(g, "deduct_limit") and g.deduct_limit
+)
 def login():
+    next_url = sanitize_next_url(request.args.get("next"))
+
     if current_user.is_authenticated:
-        LOG.d("user is already authenticated, redirect to dashboard")
-        return redirect(url_for("dashboard.index"))
+        if next_url:
+            LOG.d("user is already authenticated, redirect to %s", next_url)
+            return redirect(next_url)
+        else:
+            LOG.d("user is already authenticated, redirect to dashboard")
+            return redirect(url_for("dashboard.index"))
 
     form = LoginForm(request.form)
-    next_url = request.args.get("next")
+
     show_resend_activation = False
 
     if form.validate_on_submit():
-        user = User.filter_by(email=form.email.data).first()
+        email = sanitize_email(form.email.data)
+        canonical_email = canonicalize_email(email)
+        user = User.get_by(email=email) or User.get_by(email=canonical_email)
 
-        if not user:
-            flash("Email or password incorrect", "error")
-        elif not user.check_password(form.password.data):
+        if not user or not user.check_password(form.password.data):
+            # Trigger rate limiter
+            g.deduct_limit = True
+            form.password.data = None
             flash("Email or password incorrect", "error")
+            LoginEvent(LoginEvent.ActionType.failed).send()
+        elif user.disabled:
+            flash(
+                "Your account is disabled. Please contact SimpleLogin team to re-enable your account.",
+                "error",
+            )
+            LoginEvent(LoginEvent.ActionType.disabled_login).send()
+        elif user.delete_on is not None:
+            flash(
+                f"Your account is scheduled to be deleted on {user.delete_on}",
+                "error",
+            )
+            LoginEvent(LoginEvent.ActionType.scheduled_to_be_deleted).send()
         elif not user.activated:
             show_resend_activation = True
             flash(
                 "Please check your inbox for the activation email. You can also have this email re-sent",
                 "error",
             )
+            LoginEvent(LoginEvent.ActionType.not_activated).send()
         else:
+            LoginEvent(LoginEvent.ActionType.success).send()
             return after_login(user, next_url)
 
     return render_template(
@@ -45,4 +76,7 @@ def login():
         form=form,
         next_url=next_url,
         show_resend_activation=show_resend_activation,
+        connect_with_proton=CONNECT_WITH_PROTON,
+        connect_with_oidc=OIDC_CLIENT_ID is not None,
+        connect_with_oidc_icon=CONNECT_WITH_OIDC_ICON,
     )

app/auth/views/login_utils.py

@@ -1,30 +1,68 @@
-from flask import session, redirect, url_for
+from time import time
+from typing import Optional
+
+from flask import session, redirect, url_for, request
 from flask_login import login_user
 
 from app.config import MFA_USER_ID
 from app.log import LOG
+from app.models import Referral
 
 
-def after_login(user, next_url):
+def after_login(user, next_url, login_from_proton: bool = False):
     """
     Redirect to the correct page after login.
+    If the user is logged in with Proton, do not look at fido nor otp
     If user enables MFA: redirect user to MFA page
     Otherwise redirect to dashboard page if no next_url
     """
-    if user.enable_otp:
-        session[MFA_USER_ID] = user.id
-        if next_url:
-            return redirect(url_for("auth.mfa", next_url=next_url))
-        else:
-            return redirect(url_for("auth.mfa"))
-    else:
-        LOG.debug("log user %s in", user)
-        login_user(user)
+    if not login_from_proton:
+        if user.fido_enabled():
+            # Use the same session for FIDO so that we can easily
+            # switch between these two 2FA option
+            session[MFA_USER_ID] = user.id
+            if next_url:
+                return redirect(url_for("auth.fido", next=next_url))
+            else:
+                return redirect(url_for("auth.fido"))
+        elif user.enable_otp:
+            session[MFA_USER_ID] = user.id
+            if next_url:
+                return redirect(url_for("auth.mfa", next=next_url))
+            else:
+                return redirect(url_for("auth.mfa"))
 
-        # User comes to login page from another page
-        if next_url:
-            LOG.debug("redirect user to %s", next_url)
-            return redirect(next_url)
-        else:
-            LOG.debug("redirect user to dashboard")
-            return redirect(url_for("dashboard.index"))
+    LOG.d("log user %s in", user)
+    login_user(user)
+    session["sudo_time"] = int(time())
+
+    # User comes to login page from another page
+    if next_url:
+        LOG.d("redirect user to %s", next_url)
+        return redirect(next_url)
+    else:
+        LOG.d("redirect user to dashboard")
+        return redirect(url_for("dashboard.index"))
+
+
+# name of the cookie that stores the referral code
+_REFERRAL_COOKIE = "slref"
+
+
+def get_referral() -> Optional[Referral]:
+    """Get the eventual referral stored in cookie"""
+    # whether user arrives via a referral
+    referral = None
+    if request.cookies:
+        ref_code = request.cookies.get(_REFERRAL_COOKIE)
+        referral = Referral.get_by(code=ref_code)
+
+    if not referral:
+        if "slref" in session:
+            ref_code = session["slref"]
+            referral = Referral.get_by(code=ref_code)
+
+    if referral:
+        LOG.d("referral found %s", referral)
+
+    return referral

app/auth/views/logout.py

@@ -1,10 +1,17 @@
-from flask import render_template
-from flask_login import logout_user
+from flask import redirect, url_for, flash, make_response
 
 from app.auth.base import auth_bp
+from app.config import SESSION_COOKIE_NAME
+from app.session import logout_session
 
 
 @auth_bp.route("/logout")
 def logout():
-    logout_user()
-    return render_template("auth/logout.html")
+    logout_session()
+    flash("You are logged out", "success")
+    response = make_response(redirect(url_for("auth.login")))
+    response.delete_cookie(SESSION_COOKIE_NAME)
+    response.delete_cookie("mfa")
+    response.delete_cookie("dark-mode")
+
+    return response

app/auth/views/mfa.py

@@ -1,20 +1,38 @@
 import pyotp
-from flask import request, render_template, redirect, url_for, flash, session
+from flask import (
+    render_template,
+    redirect,
+    url_for,
+    flash,
+    session,
+    make_response,
+    request,
+    g,
+)
 from flask_login import login_user
 from flask_wtf import FlaskForm
-from wtforms import StringField, validators
+from wtforms import BooleanField, StringField, validators
 
 from app.auth.base import auth_bp
-from app.config import MFA_USER_ID
-from app.log import LOG
-from app.models import User
+from app.config import MFA_USER_ID, URL
+from app.db import Session
+from app.email_utils import send_invalid_totp_login_email
+from app.extensions import limiter
+from app.models import User, MfaBrowser
+from app.utils import sanitize_next_url
 
 
 class OtpTokenForm(FlaskForm):
     token = StringField("Token", validators=[validators.DataRequired()])
+    remember = BooleanField(
+        "attr", default=False, description="Remember this browser for 30 days"
+    )
 
 
 @auth_bp.route("/mfa", methods=["GET", "POST"])
+@limiter.limit(
+    "10/minute", deduct_when=lambda r: hasattr(g, "deduct_limit") and g.deduct_limit
+)
 def mfa():
     # passed from login page
     user_id = session.get(MFA_USER_ID)
@@ -31,28 +49,59 @@ def mfa():
         return redirect(url_for("auth.login"))
 
     otp_token_form = OtpTokenForm()
-    next_url = request.args.get("next")
+    next_url = sanitize_next_url(request.args.get("next"))
+
+    if request.cookies.get("mfa"):
+        browser = MfaBrowser.get_by(token=request.cookies.get("mfa"))
+        if browser and not browser.is_expired() and browser.user_id == user.id:
+            login_user(user)
+            flash("Welcome back!", "success")
+            # Redirect user to correct page
+            return redirect(next_url or url_for("dashboard.index"))
+        else:
+            # Trigger rate limiter
+            g.deduct_limit = True
 
     if otp_token_form.validate_on_submit():
         totp = pyotp.TOTP(user.otp_secret)
 
-        token = otp_token_form.token.data
+        token = otp_token_form.token.data.replace(" ", "")
 
-        if totp.verify(token):
+        if totp.verify(token, valid_window=2) and user.last_otp != token:
             del session[MFA_USER_ID]
+            user.last_otp = token
+            Session.commit()
 
             login_user(user)
-            flash(f"Welcome back {user.name}!")
+            flash("Welcome back!", "success")
 
-            # User comes to login page from another page
-            if next_url:
-                LOG.debug("redirect user to %s", next_url)
-                return redirect(next_url)
-            else:
-                LOG.debug("redirect user to dashboard")
-                return redirect(url_for("dashboard.index"))
+            # Redirect user to correct page
+            response = make_response(redirect(next_url or url_for("dashboard.index")))
+
+            if otp_token_form.remember.data:
+                browser = MfaBrowser.create_new(user=user)
+                Session.commit()
+                response.set_cookie(
+                    "mfa",
+                    value=browser.token,
+                    expires=browser.expires.datetime,
+                    secure=True if URL.startswith("https") else False,
+                    httponly=True,
+                    samesite="Lax",
+                )
+
+            return response
 
         else:
             flash("Incorrect token", "warning")
+            # Trigger rate limiter
+            g.deduct_limit = True
+            otp_token_form.token.data = None
+            send_invalid_totp_login_email(user, "TOTP")
 
-    return render_template("auth/mfa.html", otp_token_form=otp_token_form)
+    return render_template(
+        "auth/mfa.html",
+        otp_token_form=otp_token_form,
+        enable_fido=(user.fido_enabled()),
+        next_url=next_url,
+    )

app/auth/views/oidc.py

@@ -0,0 +1,135 @@
+from flask import request, session, redirect, flash, url_for
+from requests_oauthlib import OAuth2Session
+
+import requests
+
+from app import config
+from app.auth.base import auth_bp
+from app.auth.views.login_utils import after_login
+from app.config import (
+    URL,
+    OIDC_SCOPES,
+    OIDC_NAME_FIELD,
+)
+from app.db import Session
+from app.email_utils import send_welcome_email
+from app.log import LOG
+from app.models import User, SocialAuth
+from app.utils import sanitize_email, sanitize_next_url
+
+
+# need to set explicitly redirect_uri instead of leaving the lib to pre-fill redirect_uri
+# when served behind nginx, the redirect_uri is localhost... and not the real url
+redirect_uri = URL + "/auth/oidc/callback"
+
+SESSION_STATE_KEY = "oauth_state"
+SESSION_NEXT_KEY = "oauth_redirect_next"
+
+
+@auth_bp.route("/oidc/login")
+def oidc_login():
+    if config.OIDC_CLIENT_ID is None or config.OIDC_CLIENT_SECRET is None:
+        return redirect(url_for("auth.login"))
+
+    next_url = sanitize_next_url(request.args.get("next"))
+
+    auth_url = requests.get(config.OIDC_WELL_KNOWN_URL).json()["authorization_endpoint"]
+
+    oidc = OAuth2Session(
+        config.OIDC_CLIENT_ID, scope=[OIDC_SCOPES], redirect_uri=redirect_uri
+    )
+    authorization_url, state = oidc.authorization_url(auth_url)
+
+    # State is used to prevent CSRF, keep this for later.
+    session[SESSION_STATE_KEY] = state
+    session[SESSION_NEXT_KEY] = next_url
+    return redirect(authorization_url)
+
+
+@auth_bp.route("/oidc/callback")
+def oidc_callback():
+    if SESSION_STATE_KEY not in session:
+        flash("Invalid state, please retry", "error")
+        return redirect(url_for("auth.login"))
+    if config.OIDC_CLIENT_ID is None or config.OIDC_CLIENT_SECRET is None:
+        return redirect(url_for("auth.login"))
+
+    # user clicks on cancel
+    if "error" in request.args:
+        flash("Please use another sign in method then", "warning")
+        return redirect("/")
+
+    oidc_configuration = requests.get(config.OIDC_WELL_KNOWN_URL).json()
+    user_info_url = oidc_configuration["userinfo_endpoint"]
+    token_url = oidc_configuration["token_endpoint"]
+
+    oidc = OAuth2Session(
+        config.OIDC_CLIENT_ID,
+        state=session[SESSION_STATE_KEY],
+        scope=[OIDC_SCOPES],
+        redirect_uri=redirect_uri,
+    )
+    oidc.fetch_token(
+        token_url,
+        client_secret=config.OIDC_CLIENT_SECRET,
+        authorization_response=request.url,
+    )
+
+    oidc_user_data = oidc.get(user_info_url)
+    if oidc_user_data.status_code != 200:
+        LOG.e(
+            f"cannot get oidc user data {oidc_user_data.status_code} {oidc_user_data.text}"
+        )
+        flash(
+            "Cannot get user data from OIDC, please use another way to login/sign up",
+            "error",
+        )
+        return redirect(url_for("auth.login"))
+    oidc_user_data = oidc_user_data.json()
+
+    email = oidc_user_data.get("email")
+
+    if not email:
+        LOG.e(f"cannot get email for OIDC user {oidc_user_data} {email}")
+        flash(
+            "Cannot get a valid email from OIDC, please another way to login/sign up",
+            "error",
+        )
+        return redirect(url_for("auth.login"))
+
+    email = sanitize_email(email)
+    user = User.get_by(email=email)
+
+    if not user and config.DISABLE_REGISTRATION:
+        flash(
+            "Sorry you cannot sign up via the OIDC provider. Please sign-up first with your email.",
+            "error",
+        )
+        return redirect(url_for("auth.register"))
+    elif not user:
+        user = create_user(email, oidc_user_data)
+
+    if not SocialAuth.get_by(user_id=user.id, social="oidc"):
+        SocialAuth.create(user_id=user.id, social="oidc")
+        Session.commit()
+
+    # The activation link contains the original page, for ex authorize page
+    next_url = session[SESSION_NEXT_KEY]
+    session[SESSION_NEXT_KEY] = None
+
+    return after_login(user, next_url)
+
+
+def create_user(email, oidc_user_data):
+    new_user = User.create(
+        email=email,
+        name=oidc_user_data.get(OIDC_NAME_FIELD),
+        password="",
+        activated=True,
+    )
+    LOG.i(f"Created new user for login request from OIDC. New user {new_user.id}")
+    Session.commit()
+
+    send_welcome_email(new_user)
+
+    return new_user

app/auth/views/proton.py

@@ -0,0 +1,190 @@
+import requests
+from flask import request, session, redirect, flash, url_for
+from flask_limiter.util import get_remote_address
+from flask_login import current_user
+from requests_oauthlib import OAuth2Session
+from typing import Optional
+
+from app.auth.base import auth_bp
+from app.auth.views.login_utils import after_login
+from app.config import (
+    PROTON_BASE_URL,
+    PROTON_CLIENT_ID,
+    PROTON_CLIENT_SECRET,
+    PROTON_EXTRA_HEADER_NAME,
+    PROTON_EXTRA_HEADER_VALUE,
+    PROTON_VALIDATE_CERTS,
+    URL,
+)
+from app.log import LOG
+from app.models import ApiKey, User
+from app.proton.proton_client import HttpProtonClient, convert_access_token
+from app.proton.proton_callback_handler import (
+    ProtonCallbackHandler,
+    Action,
+)
+from app.proton.utils import get_proton_partner
+from app.utils import sanitize_next_url, sanitize_scheme
+
+_authorization_base_url = PROTON_BASE_URL + "/oauth/authorize"
+_token_url = PROTON_BASE_URL + "/oauth/token"
+
+# need to set explicitly redirect_uri instead of leaving the lib to pre-fill redirect_uri
+# when served behind nginx, the redirect_uri is localhost... and not the real url
+_redirect_uri = URL + "/auth/proton/callback"
+
+SESSION_ACTION_KEY = "oauth_action"
+SESSION_STATE_KEY = "oauth_state"
+DEFAULT_SCHEME = "auth.simplelogin"
+
+
+def get_api_key_for_user(user: User) -> str:
+    ak = ApiKey.create(
+        user_id=user.id,
+        name="Created via Login with Proton on mobile app",
+        commit=True,
+    )
+    return ak.code
+
+
+def extract_action() -> Optional[Action]:
+    action = request.args.get("action")
+    if action is not None:
+        if action == "link":
+            return Action.Link
+        elif action == "login":
+            return Action.Login
+        else:
+            LOG.w(f"Unknown action received: {action}")
+            return None
+    return Action.Login
+
+
+def get_action_from_state() -> Action:
+    oauth_action = session[SESSION_ACTION_KEY]
+    if oauth_action == Action.Login.value:
+        return Action.Login
+    elif oauth_action == Action.Link.value:
+        return Action.Link
+    raise Exception(f"Unknown action in state: {oauth_action}")
+
+
+@auth_bp.route("/proton/login")
+def proton_login():
+    if PROTON_CLIENT_ID is None or PROTON_CLIENT_SECRET is None:
+        return redirect(url_for("auth.login"))
+
+    action = extract_action()
+    if action is None:
+        return redirect(url_for("auth.login"))
+    if action == Action.Link and not current_user.is_authenticated:
+        return redirect(url_for("auth.login"))
+
+    next_url = sanitize_next_url(request.args.get("next"))
+    if next_url:
+        session["oauth_next"] = next_url
+    elif "oauth_next" in session:
+        del session["oauth_next"]
+
+    scheme = sanitize_scheme(request.args.get("scheme"))
+    if scheme:
+        session["oauth_scheme"] = scheme
+    elif "oauth_scheme" in session:
+        del session["oauth_scheme"]
+
+    mode = request.args.get("mode", "session")
+    if mode == "apikey":
+        session["oauth_mode"] = "apikey"
+    else:
+        session["oauth_mode"] = "session"
+
+    proton = OAuth2Session(PROTON_CLIENT_ID, redirect_uri=_redirect_uri)
+    authorization_url, state = proton.authorization_url(_authorization_base_url)
+
+    # State is used to prevent CSRF, keep this for later.
+    session[SESSION_STATE_KEY] = state
+    session[SESSION_ACTION_KEY] = action.value
+    return redirect(authorization_url)
+
+
+@auth_bp.route("/proton/callback")
+def proton_callback():
+    if SESSION_STATE_KEY not in session or SESSION_STATE_KEY not in session:
+        flash("Invalid state, please retry", "error")
+        return redirect(url_for("auth.login"))
+    if PROTON_CLIENT_ID is None or PROTON_CLIENT_SECRET is None:
+        return redirect(url_for("auth.login"))
+
+    # user clicks on cancel
+    if "error" in request.args:
+        flash("Please use another sign in method then", "warning")
+        return redirect("/")
+
+    proton = OAuth2Session(
+        PROTON_CLIENT_ID,
+        state=session[SESSION_STATE_KEY],
+        redirect_uri=_redirect_uri,
+    )
+
+    def check_status_code(response: requests.Response) -> requests.Response:
+        if response.status_code != 200:
+            raise Exception(
+                f"Bad Proton API response [status={response.status_code}]: {response.json()}"
+            )
+        return response
+
+    proton.register_compliance_hook("access_token_response", check_status_code)
+
+    headers = None
+    if PROTON_EXTRA_HEADER_NAME and PROTON_EXTRA_HEADER_VALUE:
+        headers = {PROTON_EXTRA_HEADER_NAME: PROTON_EXTRA_HEADER_VALUE}
+
+    try:
+        token = proton.fetch_token(
+            _token_url,
+            client_secret=PROTON_CLIENT_SECRET,
+            authorization_response=request.url,
+            verify=PROTON_VALIDATE_CERTS,
+            method="GET",
+            include_client_id=True,
+            headers=headers,
+        )
+    except Exception as e:
+        LOG.warning(f"Error fetching Proton token: {e}")
+        flash("There was an error in the login process", "error")
+        return redirect(url_for("auth.login"))
+
+    credentials = convert_access_token(token["access_token"])
+    action = get_action_from_state()
+
+    proton_client = HttpProtonClient(
+        PROTON_BASE_URL, credentials, get_remote_address(), verify=PROTON_VALIDATE_CERTS
+    )
+    handler = ProtonCallbackHandler(proton_client)
+    proton_partner = get_proton_partner()
+
+    next_url = session.get("oauth_next")
+    if action == Action.Login:
+        res = handler.handle_login(proton_partner)
+    elif action == Action.Link:
+        res = handler.handle_link(current_user, proton_partner)
+    else:
+        raise Exception(f"Unknown Action: {action.name}")
+
+    if res.flash_message is not None:
+        flash(res.flash_message, res.flash_category)
+
+    oauth_scheme = session.get("oauth_scheme")
+    if session.get("oauth_mode", "session") == "apikey":
+        apikey = get_api_key_for_user(res.user)
+        scheme = oauth_scheme or DEFAULT_SCHEME
+        return redirect(f"{scheme}:///login?apikey={apikey}")
+
+    if res.redirect_to_login:
+        return redirect(url_for("auth.login"))
+
+    if next_url and next_url[0] == "/" and oauth_scheme:
+        next_url = f"{oauth_scheme}://{next_url}"
+
+    redirect_url = next_url or res.redirect
+    return after_login(res.user, redirect_url, login_from_proton=True)

app/auth/views/recovery.py

@@ -0,0 +1,75 @@
+import arrow
+from flask import request, render_template, redirect, url_for, flash, session, g
+from flask_login import login_user
+from flask_wtf import FlaskForm
+from wtforms import StringField, validators
+
+from app.auth.base import auth_bp
+from app.config import MFA_USER_ID
+from app.db import Session
+from app.email_utils import send_invalid_totp_login_email
+from app.extensions import limiter
+from app.log import LOG
+from app.models import User, RecoveryCode
+from app.utils import sanitize_next_url
+
+
+class RecoveryForm(FlaskForm):
+    code = StringField("Code", validators=[validators.DataRequired()])
+
+
+@auth_bp.route("/recovery", methods=["GET", "POST"])
+@limiter.limit(
+    "10/minute", deduct_when=lambda r: hasattr(g, "deduct_limit") and g.deduct_limit
+)
+def recovery_route():
+    # passed from login page
+    user_id = session.get(MFA_USER_ID)
+
+    # user access this page directly without passing by login page
+    if not user_id:
+        flash("Unknown error, redirect back to main page", "warning")
+        return redirect(url_for("auth.login"))
+
+    user = User.get(user_id)
+
+    if not user.two_factor_authentication_enabled():
+        flash("Only user with MFA enabled should go to this page", "warning")
+        return redirect(url_for("auth.login"))
+
+    recovery_form = RecoveryForm()
+    next_url = sanitize_next_url(request.args.get("next"))
+
+    if recovery_form.validate_on_submit():
+        code = recovery_form.code.data
+        recovery_code = RecoveryCode.find_by_user_code(user, code)
+
+        if recovery_code:
+            if recovery_code.used:
+                # Trigger rate limiter
+                g.deduct_limit = True
+                flash("Code already used", "error")
+            else:
+                del session[MFA_USER_ID]
+
+                login_user(user)
+                flash("Welcome back!", "success")
+
+                recovery_code.used = True
+                recovery_code.used_at = arrow.now()
+                Session.commit()
+
+                # User comes to login page from another page
+                if next_url:
+                    LOG.d("redirect user to %s", next_url)
+                    return redirect(next_url)
+                else:
+                    LOG.d("redirect user to dashboard")
+                    return redirect(url_for("dashboard.index"))
+        else:
+            # Trigger rate limiter
+            g.deduct_limit = True
+            flash("Incorrect code", "error")
+            send_invalid_totp_login_email(user, "recovery")
+
+    return render_template("auth/recovery.html", recovery_form=recovery_form)

app/auth/views/register.py

@@ -1,3 +1,4 @@
+import requests
 from flask import request, flash, render_template, redirect, url_for
 from flask_login import current_user
 from flask_wtf import FlaskForm
@@ -5,18 +6,25 @@ from wtforms import StringField, validators
 
 from app import email_utils, config
 from app.auth.base import auth_bp
-from app.config import URL, DISABLE_REGISTRATION
-from app.email_utils import can_be_used_as_personal_email, email_already_used
-from app.extensions import db
+from app.config import CONNECT_WITH_PROTON, CONNECT_WITH_OIDC_ICON
+from app.auth.views.login_utils import get_referral
+from app.config import URL, HCAPTCHA_SECRET, HCAPTCHA_SITEKEY
+from app.db import Session
+from app.email_utils import (
+    email_can_be_used_as_mailbox,
+    personal_email_already_used,
+)
+from app.events.auth_event import RegisterEvent
 from app.log import LOG
-from app.models import User, ActivationCode
-from app.utils import random_string, encode_url
+from app.models import User, ActivationCode, DailyMetric
+from app.utils import random_string, encode_url, sanitize_email, canonicalize_email
 
 
 class RegisterForm(FlaskForm):
     email = StringField("Email", validators=[validators.DataRequired()])
     password = StringField(
-        "Password", validators=[validators.DataRequired(), validators.Length(min=8)]
+        "Password",
+        validators=[validators.DataRequired(), validators.Length(min=8, max=100)],
     )
 
 
@@ -35,28 +43,81 @@ def register():
     next_url = request.args.get("next")
 
     if form.validate_on_submit():
-        email = form.email.data.lower()
-        if not can_be_used_as_personal_email(email):
-            flash("You cannot use this email address as your personal inbox.", "error")
-        else:
-            if email_already_used(email):
-                flash(f"Email {email} already used", "error")
-            else:
-                LOG.debug("create user %s", form.email.data)
-                user = User.create(email=email, name="", password=form.password.data)
-                db.session.commit()
+        # only check if hcaptcha is enabled
+        if HCAPTCHA_SECRET:
+            # check with hCaptcha
+            token = request.form.get("h-captcha-response")
+            params = {"secret": HCAPTCHA_SECRET, "response": token}
+            hcaptcha_res = requests.post(
+                "https://hcaptcha.com/siteverify", data=params
+            ).json()
+            # return something like
+            # {'success': True,
+            #  'challenge_ts': '2020-07-23T10:03:25',
+            #  'hostname': '127.0.0.1'}
+            if not hcaptcha_res["success"]:
+                LOG.w(
+                    "User put wrong captcha %s %s",
+                    form.email.data,
+                    hcaptcha_res,
+                )
+                flash("Wrong Captcha", "error")
+                RegisterEvent(RegisterEvent.ActionType.catpcha_failed).send()
+                return render_template(
+                    "auth/register.html",
+                    form=form,
+                    next_url=next_url,
+                    HCAPTCHA_SITEKEY=HCAPTCHA_SITEKEY,
+                )
 
-                send_activation_email(user, next_url)
+        email = canonicalize_email(form.email.data)
+        if not email_can_be_used_as_mailbox(email):
+            flash("You cannot use this email address as your personal inbox.", "error")
+            RegisterEvent(RegisterEvent.ActionType.email_in_use).send()
+        else:
+            sanitized_email = sanitize_email(form.email.data)
+            if personal_email_already_used(email) or personal_email_already_used(
+                sanitized_email
+            ):
+                flash(f"Email {email} already used", "error")
+                RegisterEvent(RegisterEvent.ActionType.email_in_use).send()
+            else:
+                LOG.d("create user %s", email)
+                user = User.create(
+                    email=email,
+                    name=form.email.data,
+                    password=form.password.data,
+                    referral=get_referral(),
+                )
+                Session.commit()
+
+                try:
+                    send_activation_email(user, next_url)
+                    RegisterEvent(RegisterEvent.ActionType.success).send()
+                    DailyMetric.get_or_create_today_metric().nb_new_web_non_proton_user += 1
+                    Session.commit()
+                except Exception:
+                    flash("Invalid email, are you sure the email is correct?", "error")
+                    RegisterEvent(RegisterEvent.ActionType.invalid_email).send()
+                    return redirect(url_for("auth.register"))
 
                 return render_template("auth/register_waiting_activation.html")
 
-    return render_template("auth/register.html", form=form, next_url=next_url)
+    return render_template(
+        "auth/register.html",
+        form=form,
+        next_url=next_url,
+        HCAPTCHA_SITEKEY=HCAPTCHA_SITEKEY,
+        connect_with_proton=CONNECT_WITH_PROTON,
+        connect_with_oidc=config.OIDC_CLIENT_ID is not None,
+        connect_with_oidc_icon=CONNECT_WITH_OIDC_ICON,
+    )
 
 
 def send_activation_email(user, next_url):
     # the activation code is valid for 1h
     activation = ActivationCode.create(user_id=user.id, code=random_string(30))
-    db.session.commit()
+    Session.commit()
 
     # Send user activation email
     activation_link = f"{URL}/auth/activate?code={activation.code}"
@@ -64,4 +125,4 @@ def send_activation_email(user, next_url):
         LOG.d("redirect user to %s after activation", next_url)
         activation_link = activation_link + "&next=" + encode_url(next_url)
 
-    email_utils.send_activation_email(user.email, user.name, activation_link)
+    email_utils.send_activation_email(user.email, activation_link)

app/auth/views/resend_activation.py

@@ -4,8 +4,10 @@ from wtforms import StringField, validators
 
 from app.auth.base import auth_bp
 from app.auth.views.register import send_activation_email
+from app.extensions import limiter
 from app.log import LOG
 from app.models import User
+from app.utils import sanitize_email, canonicalize_email
 
 
 class ResendActivationForm(FlaskForm):
@@ -13,11 +15,14 @@ class ResendActivationForm(FlaskForm):
 
 
 @auth_bp.route("/resend_activation", methods=["GET", "POST"])
+@limiter.limit("10/hour")
 def resend_activation():
     form = ResendActivationForm(request.form)
 
     if form.validate_on_submit():
-        user = User.filter_by(email=form.email.data).first()
+        email = sanitize_email(form.email.data)
+        canonical_email = canonicalize_email(email)
+        user = User.get_by(email=email) or User.get_by(email=canonical_email)
 
         if not user:
             flash("There is no such email", "warning")

app/auth/views/reset_password.py

@@ -1,21 +1,27 @@
-import arrow
-from flask import request, flash, render_template, redirect, url_for
-from flask_login import login_user
+import uuid
+
+from flask import request, flash, render_template, url_for, g
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators
 
 from app.auth.base import auth_bp
-from app.extensions import db
+from app.auth.views.login_utils import after_login
+from app.db import Session
+from app.extensions import limiter
 from app.models import ResetPasswordCode
 
 
 class ResetPasswordForm(FlaskForm):
     password = StringField(
-        "Password", validators=[validators.DataRequired(), validators.Length(min=8)]
+        "Password",
+        validators=[validators.DataRequired(), validators.Length(min=8, max=100)],
     )
 
 
 @auth_bp.route("/reset_password", methods=["GET", "POST"])
+@limiter.limit(
+    "10/minute", deduct_when=lambda r: hasattr(g, "deduct_limit") and g.deduct_limit
+)
 def reset_password():
     form = ResetPasswordForm(request.form)
 
@@ -26,6 +32,8 @@ def reset_password():
     )
 
     if not reset_password_code:
+        # Trigger rate limiter
+        g.deduct_limit = True
         error = (
             "The reset password link can be used only once. "
             "Please request a new link to reset password."
@@ -38,20 +46,30 @@ def reset_password():
 
     if form.validate_on_submit():
         user = reset_password_code.user
+        new_password = form.password.data
 
-        user.set_password(form.password.data)
+        # avoid user reusing the old password
+        if user.check_password(new_password):
+            error = "You cannot reuse the same password"
+            return render_template("auth/reset_password.html", form=form, error=error)
+
+        user.set_password(new_password)
 
         flash("Your new password has been set", "success")
 
         # this can be served to activate user too
         user.activated = True
 
-        # remove the reset password code
-        ResetPasswordCode.delete(reset_password_code.id)
+        # remove all reset password codes
+        ResetPasswordCode.filter_by(user_id=user.id).delete()
 
-        db.session.commit()
-        login_user(user)
+        # change the alternative_id to log user out on other browsers
+        user.alternative_id = str(uuid.uuid4())
 
-        return redirect(url_for("dashboard.index"))
+        Session.commit()
+
+        # do not use login_user(user) here
+        # to make sure user needs to go through MFA if enabled
+        return after_login(user, url_for("dashboard.index"))
 
     return render_template("auth/reset_password.html", form=form)

app/auth/views/social.py

@@ -0,0 +1,14 @@
+from flask import render_template, redirect, url_for
+from flask_login import current_user
+
+from app.auth.base import auth_bp
+from app.log import LOG
+
+
+@auth_bp.route("/social", methods=["GET", "POST"])
+def social():
+    if current_user.is_authenticated:
+        LOG.d("user is already authenticated, redirect to dashboard")
+        return redirect(url_for("dashboard.index"))
+
+    return render_template("auth/social.html")

app/build_info.py

@@ -0,0 +1,2 @@
+SHA1 = "dev"
+BUILD_TIME = "1652365083"

app/config.py

@@ -1,13 +1,13 @@
 import os
 import random
+import socket
 import string
-import subprocess
-import tempfile
-from uuid import uuid4
+from ast import literal_eval
+from typing import Callable, List
+from urllib.parse import urlparse
 
 from dotenv import load_dotenv
 
-SHA1 = subprocess.getoutput("git rev-parse HEAD")
 ROOT_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 
 
@@ -20,6 +20,21 @@ def get_abs_path(file_path: str):
         return os.path.join(ROOT_DIR, file_path)
 
 
+def sl_getenv(env_var: str, default_factory: Callable = None):
+    """
+    Get env value, convert into Python object
+    Args:
+        env_var (str): env var, example: SL_DB
+        default_factory: returns value if this env var is not set.
+
+    """
+    value = os.getenv(env_var)
+    if value is None:
+        return default_factory()
+
+    return literal_eval(value)
+
+
 config_file = os.environ.get("CONFIG")
 if config_file:
     config_file = get_abs_path(config_file)
@@ -28,18 +43,18 @@ if config_file:
 else:
     load_dotenv()
 
-RESET_DB = "RESET_DB" in os.environ
 COLOR_LOG = "COLOR_LOG" in os.environ
 
 # Allow user to have 1 year of premium: set the expiration_date to 1 year more
 PROMO_CODE = "SIMPLEISBETTER"
 
-# Debug mode
-DEBUG = os.environ["DEBUG"] if "DEBUG" in os.environ else False
 # Server url
 URL = os.environ["URL"]
 print(">>> URL:", URL)
 
+# Calculate RP_ID for WebAuthn
+RP_ID = urlparse(URL).hostname
+
 SENTRY_DSN = os.environ.get("SENTRY_DSN")
 
 # can use another sentry project for the front-end to avoid noises
@@ -47,105 +62,151 @@ SENTRY_FRONT_END_DSN = os.environ.get("SENTRY_FRONT_END_DSN") or SENTRY_DSN
 
 # Email related settings
 NOT_SEND_EMAIL = "NOT_SEND_EMAIL" in os.environ
-EMAIL_DOMAIN = os.environ["EMAIL_DOMAIN"]
+EMAIL_DOMAIN = os.environ["EMAIL_DOMAIN"].lower()
 SUPPORT_EMAIL = os.environ["SUPPORT_EMAIL"]
 SUPPORT_NAME = os.environ.get("SUPPORT_NAME", "Son from SimpleLogin")
 ADMIN_EMAIL = os.environ.get("ADMIN_EMAIL")
+# to receive monitoring daily report
+MONITORING_EMAIL = os.environ.get("MONITORING_EMAIL")
+
+# VERP: mail_from set to BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX
+BOUNCE_PREFIX = os.environ.get("BOUNCE_PREFIX") or "bounce+"
+BOUNCE_SUFFIX = os.environ.get("BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
+
+# Used for VERP during reply phase. It's similar to BOUNCE_PREFIX.
+# It's needed when sending emails from custom domain to respect DMARC.
+# BOUNCE_PREFIX_FOR_REPLY_PHASE should never be used in any existing alias
+# and can't be used for creating a new alias on custom domain
+# Note BOUNCE_PREFIX_FOR_REPLY_PHASE doesn't have the trailing plus sign (+) as BOUNCE_PREFIX
+BOUNCE_PREFIX_FOR_REPLY_PHASE = (
+    os.environ.get("BOUNCE_PREFIX_FOR_REPLY_PHASE") or "bounce_reply"
+)
+
+# VERP for transactional email: mail_from set to BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX
+TRANSACTIONAL_BOUNCE_PREFIX = (
+    os.environ.get("TRANSACTIONAL_BOUNCE_PREFIX") or "transactional+"
+)
+TRANSACTIONAL_BOUNCE_SUFFIX = (
+    os.environ.get("TRANSACTIONAL_BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
+)
+
 try:
     MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"])
 except Exception:
     print("MAX_NB_EMAIL_FREE_PLAN is not set, use 5 as default value")
     MAX_NB_EMAIL_FREE_PLAN = 5
 
-# allow to override postfix server locally
+MAX_NB_EMAIL_OLD_FREE_PLAN = int(os.environ.get("MAX_NB_EMAIL_OLD_FREE_PLAN", 15))
+
+# maximum number of directory a premium user can create
+MAX_NB_DIRECTORY = 50
+MAX_NB_SUBDOMAIN = 5
+
+ENFORCE_SPF = "ENFORCE_SPF" in os.environ
+
+# override postfix server locally
+# use 240.0.0.1 here instead of 10.0.0.1 as existing SL instances use the 240.0.0.0 network
 POSTFIX_SERVER = os.environ.get("POSTFIX_SERVER", "240.0.0.1")
 
 DISABLE_REGISTRATION = "DISABLE_REGISTRATION" in os.environ
 
+# allow using a different postfix port, useful when developing locally
+
 # Use port 587 instead of 25 when sending emails through Postfix
 # Useful when calling Postfix from an external network
 POSTFIX_SUBMISSION_TLS = "POSTFIX_SUBMISSION_TLS" in os.environ
-
-if "OTHER_ALIAS_DOMAINS" in os.environ:
-    OTHER_ALIAS_DOMAINS = eval(
-        os.environ["OTHER_ALIAS_DOMAINS"]
-    )  # ["domain1.com", "domain2.com"]
+if POSTFIX_SUBMISSION_TLS:
+    default_postfix_port = 587
 else:
-    OTHER_ALIAS_DOMAINS = []
+    default_postfix_port = 25
+POSTFIX_PORT = int(os.environ.get("POSTFIX_PORT", default_postfix_port))
+POSTFIX_TIMEOUT = os.environ.get("POSTFIX_TIMEOUT", 3)
+
+# ["domain1.com", "domain2.com"]
+OTHER_ALIAS_DOMAINS = sl_getenv("OTHER_ALIAS_DOMAINS", list)
+OTHER_ALIAS_DOMAINS = [d.lower().strip() for d in OTHER_ALIAS_DOMAINS]
 
 # List of domains user can use to create alias
-ALIAS_DOMAINS = OTHER_ALIAS_DOMAINS + [EMAIL_DOMAIN]
+if "ALIAS_DOMAINS" in os.environ:
+    ALIAS_DOMAINS = sl_getenv("ALIAS_DOMAINS")  # ["domain1.com", "domain2.com"]
+else:
+    ALIAS_DOMAINS = OTHER_ALIAS_DOMAINS + [EMAIL_DOMAIN]
+ALIAS_DOMAINS = [d.lower().strip() for d in ALIAS_DOMAINS]
+
+# ["domain1.com", "domain2.com"]
+PREMIUM_ALIAS_DOMAINS = sl_getenv("PREMIUM_ALIAS_DOMAINS", list)
+PREMIUM_ALIAS_DOMAINS = [d.lower().strip() for d in PREMIUM_ALIAS_DOMAINS]
+
+# the alias domain used when creating the first alias for user
+FIRST_ALIAS_DOMAIN = os.environ.get("FIRST_ALIAS_DOMAIN") or EMAIL_DOMAIN
 
 # list of (priority, email server)
-EMAIL_SERVERS_WITH_PRIORITY = eval(
-    os.environ["EMAIL_SERVERS_WITH_PRIORITY"]
-)  # [(10, "email.hostname.")]
-
-# these emails are ignored when computing stats
-if os.environ.get("IGNORED_EMAILS"):
-    IGNORED_EMAILS = eval(os.environ.get("IGNORED_EMAILS"))
-else:
-    IGNORED_EMAILS = []
+# e.g. [(10, "mx1.hostname."), (10, "mx2.hostname.")]
+EMAIL_SERVERS_WITH_PRIORITY = sl_getenv("EMAIL_SERVERS_WITH_PRIORITY")
 
 # disable the alias suffix, i.e. the ".random_word" part
 DISABLE_ALIAS_SUFFIX = "DISABLE_ALIAS_SUFFIX" in os.environ
 
-DKIM_PRIVATE_KEY_PATH = get_abs_path(os.environ["DKIM_PRIVATE_KEY_PATH"])
-DKIM_PUBLIC_KEY_PATH = get_abs_path(os.environ["DKIM_PUBLIC_KEY_PATH"])
+# the email address that receives all unsubscription request
+UNSUBSCRIBER = os.environ.get("UNSUBSCRIBER")
+
+# due to a typo, both UNSUBSCRIBER and OLD_UNSUBSCRIBER are supported
+OLD_UNSUBSCRIBER = os.environ.get("OLD_UNSUBSCRIBER")
+
 DKIM_SELECTOR = b"dkim"
+DKIM_PRIVATE_KEY = None
 
-with open(DKIM_PRIVATE_KEY_PATH) as f:
-    DKIM_PRIVATE_KEY = f.read()
-
-
-with open(DKIM_PUBLIC_KEY_PATH) as f:
-    DKIM_DNS_VALUE = (
-        f.read()
-        .replace("-----BEGIN PUBLIC KEY-----", "")
-        .replace("-----END PUBLIC KEY-----", "")
-        .replace("\r", "")
-        .replace("\n", "")
-    )
-
-
-DKIM_HEADERS = [b"from", b"to", b"subject"]
+if "DKIM_PRIVATE_KEY_PATH" in os.environ:
+    DKIM_PRIVATE_KEY_PATH = get_abs_path(os.environ["DKIM_PRIVATE_KEY_PATH"])
+    with open(DKIM_PRIVATE_KEY_PATH) as f:
+        DKIM_PRIVATE_KEY = f.read()
 
 # Database
 DB_URI = os.environ["DB_URI"]
+DB_CONN_NAME = os.environ.get("DB_CONN_NAME", "webapp")
 
 # Flask secret
 FLASK_SECRET = os.environ["FLASK_SECRET"]
+if not FLASK_SECRET:
+    raise RuntimeError("FLASK_SECRET is empty. Please define it.")
+SESSION_COOKIE_NAME = "slapp"
 MAILBOX_SECRET = FLASK_SECRET + "mailbox"
+CUSTOM_ALIAS_SECRET = FLASK_SECRET + "custom_alias"
+UNSUBSCRIBE_SECRET = FLASK_SECRET + "unsub"
 
 # AWS
-AWS_REGION = "eu-west-3"
+AWS_REGION = os.environ.get("AWS_REGION") or "eu-west-3"
 BUCKET = os.environ.get("BUCKET")
 AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID")
 AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
-
-CLOUDWATCH_LOG_GROUP = CLOUDWATCH_LOG_STREAM = ""
-ENABLE_CLOUDWATCH = "ENABLE_CLOUDWATCH" in os.environ
-if ENABLE_CLOUDWATCH:
-    CLOUDWATCH_LOG_GROUP = os.environ["CLOUDWATCH_LOG_GROUP"]
-    CLOUDWATCH_LOG_STREAM = os.environ["CLOUDWATCH_LOG_STREAM"]
+AWS_ENDPOINT_URL = os.environ.get("AWS_ENDPOINT_URL", None)
 
 # Paddle
 try:
     PADDLE_VENDOR_ID = int(os.environ["PADDLE_VENDOR_ID"])
     PADDLE_MONTHLY_PRODUCT_ID = int(os.environ["PADDLE_MONTHLY_PRODUCT_ID"])
     PADDLE_YEARLY_PRODUCT_ID = int(os.environ["PADDLE_YEARLY_PRODUCT_ID"])
-except:
+except (KeyError, ValueError):
     print("Paddle param not set")
     PADDLE_VENDOR_ID = -1
     PADDLE_MONTHLY_PRODUCT_ID = -1
     PADDLE_YEARLY_PRODUCT_ID = -1
 
+# Other Paddle product IDS
+PADDLE_MONTHLY_PRODUCT_IDS = sl_getenv("PADDLE_MONTHLY_PRODUCT_IDS", list)
+PADDLE_MONTHLY_PRODUCT_IDS.append(PADDLE_MONTHLY_PRODUCT_ID)
+
+PADDLE_YEARLY_PRODUCT_IDS = sl_getenv("PADDLE_YEARLY_PRODUCT_IDS", list)
+PADDLE_YEARLY_PRODUCT_IDS.append(PADDLE_YEARLY_PRODUCT_ID)
+
 PADDLE_PUBLIC_KEY_PATH = get_abs_path(
     os.environ.get("PADDLE_PUBLIC_KEY_PATH", "local_data/paddle.key.pub")
 )
 
 PADDLE_AUTH_CODE = os.environ.get("PADDLE_AUTH_CODE")
 
+PADDLE_COUPON_ID = os.environ.get("PADDLE_COUPON_ID")
+
 # OpenID keys, used to sign id_token
 OPENID_PRIVATE_KEY_PATH = get_abs_path(
     os.environ.get("OPENID_PRIVATE_KEY_PATH", "local_data/jwtRS256.key")
@@ -155,8 +216,10 @@ OPENID_PUBLIC_KEY_PATH = get_abs_path(
 )
 
 # Used to generate random email
+# words.txt is a list of English words and doesn't contain any "bad" word
+# words_alpha.txt comes from https://github.com/dwyl/english-words and also contains bad words.
 WORDS_FILE_PATH = get_abs_path(
-    os.environ.get("WORDS_FILE_PATH", "local_data/words_alpha.txt")
+    os.environ.get("WORDS_FILE_PATH", "local_data/words.txt")
 )
 
 # Used to generate random email
@@ -171,17 +234,33 @@ else:
 
     print("WARNING: Use a temp directory for GNUPGHOME", GNUPGHOME)
 
-# Github, Google, Facebook client id and secrets
+# Github, Google, Facebook, OIDC client id and secrets
 GITHUB_CLIENT_ID = os.environ.get("GITHUB_CLIENT_ID")
 GITHUB_CLIENT_SECRET = os.environ.get("GITHUB_CLIENT_SECRET")
 
-
 GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID")
 GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_CLIENT_SECRET")
 
 FACEBOOK_CLIENT_ID = os.environ.get("FACEBOOK_CLIENT_ID")
 FACEBOOK_CLIENT_SECRET = os.environ.get("FACEBOOK_CLIENT_SECRET")
 
+CONNECT_WITH_OIDC_ICON = os.environ.get("CONNECT_WITH_OIDC_ICON")
+OIDC_WELL_KNOWN_URL = os.environ.get("OIDC_WELL_KNOWN_URL")
+OIDC_CLIENT_ID = os.environ.get("OIDC_CLIENT_ID")
+OIDC_CLIENT_SECRET = os.environ.get("OIDC_CLIENT_SECRET")
+OIDC_SCOPES = os.environ.get("OIDC_SCOPES")
+OIDC_NAME_FIELD = os.environ.get("OIDC_NAME_FIELD", "name")
+
+PROTON_CLIENT_ID = os.environ.get("PROTON_CLIENT_ID")
+PROTON_CLIENT_SECRET = os.environ.get("PROTON_CLIENT_SECRET")
+PROTON_BASE_URL = os.environ.get(
+    "PROTON_BASE_URL", "https://account.protonmail.com/api"
+)
+PROTON_VALIDATE_CERTS = "PROTON_VALIDATE_CERTS" in os.environ
+CONNECT_WITH_PROTON = "CONNECT_WITH_PROTON" in os.environ
+PROTON_EXTRA_HEADER_NAME = os.environ.get("PROTON_EXTRA_HEADER_NAME")
+PROTON_EXTRA_HEADER_VALUE = os.environ.get("PROTON_EXTRA_HEADER_VALUE")
+
 # in seconds
 AVATAR_URL_EXPIRATION = 3600 * 24 * 7  # 1h*24h/d*7d=1week
 
@@ -191,9 +270,320 @@ MFA_USER_ID = "mfa_user_id"
 FLASK_PROFILER_PATH = os.environ.get("FLASK_PROFILER_PATH")
 FLASK_PROFILER_PASSWORD = os.environ.get("FLASK_PROFILER_PASSWORD")
 
-
 # Job names
 JOB_ONBOARDING_1 = "onboarding-1"
+JOB_ONBOARDING_2 = "onboarding-2"
+JOB_ONBOARDING_3 = "onboarding-3"
+JOB_ONBOARDING_4 = "onboarding-4"
+JOB_BATCH_IMPORT = "batch-import"
+JOB_DELETE_ACCOUNT = "delete-account"
+JOB_DELETE_MAILBOX = "delete-mailbox"
+JOB_DELETE_DOMAIN = "delete-domain"
+JOB_SEND_USER_REPORT = "send-user-report"
+JOB_SEND_PROTON_WELCOME_1 = "proton-welcome-1"
 
 # for pagination
 PAGE_LIMIT = 20
+
+# Upload to static/upload instead of s3
+LOCAL_FILE_UPLOAD = "LOCAL_FILE_UPLOAD" in os.environ
+UPLOAD_DIR = None
+
+# Rate Limiting
+# nb max of activity (forward/reply) an alias can have during 1 min
+MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS = 10
+
+# nb max of activity (forward/reply) a mailbox can have during 1 min
+MAX_ACTIVITY_DURING_MINUTE_PER_MAILBOX = 15
+
+if LOCAL_FILE_UPLOAD:
+    print("Upload files to local dir")
+    UPLOAD_DIR = os.path.join(ROOT_DIR, "static/upload")
+    if not os.path.exists(UPLOAD_DIR):
+        print("Create upload dir")
+        os.makedirs(UPLOAD_DIR)
+
+LANDING_PAGE_URL = os.environ.get("LANDING_PAGE_URL") or "https://simplelogin.io"
+
+STATUS_PAGE_URL = os.environ.get("STATUS_PAGE_URL") or "https://status.simplelogin.io"
+
+# Loading PGP keys when mail_handler runs. To be used locally when init_app is not called.
+LOAD_PGP_EMAIL_HANDLER = "LOAD_PGP_EMAIL_HANDLER" in os.environ
+
+# Used when querying info on Apple API
+# for iOS App
+APPLE_API_SECRET = os.environ.get("APPLE_API_SECRET")
+# for Mac App
+MACAPP_APPLE_API_SECRET = os.environ.get("MACAPP_APPLE_API_SECRET")
+
+# <<<<< ALERT EMAIL >>>>
+
+# maximal number of alerts that can be sent to the same email in 24h
+MAX_ALERT_24H = 4
+
+# When a reverse-alias receives emails from un unknown mailbox
+ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX = "reverse_alias_unknown_mailbox"
+
+# When somebody is trying to spoof a reply
+ALERT_DMARC_FAILED_REPLY_PHASE = "dmarc_failed_reply_phase"
+
+# When a forwarding email is bounced
+ALERT_BOUNCE_EMAIL = "bounce"
+
+ALERT_BOUNCE_EMAIL_REPLY_PHASE = "bounce-when-reply"
+
+# When a forwarding email is detected as spam
+ALERT_SPAM_EMAIL = "spam"
+
+# When an email is sent from a mailbox to an alias - a cycle
+ALERT_SEND_EMAIL_CYCLE = "cycle"
+
+ALERT_NON_REVERSE_ALIAS_REPLY_PHASE = "non_reverse_alias_reply_phase"
+
+ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS = "from_address_is_reverse_alias"
+
+ALERT_TO_NOREPLY = "to_noreply"
+
+ALERT_SPF = "spf"
+
+ALERT_INVALID_TOTP_LOGIN = "invalid_totp_login"
+
+# when a mailbox is also an alias
+# happens when user adds a mailbox with their domain
+# then later adds this domain into SimpleLogin
+ALERT_MAILBOX_IS_ALIAS = "mailbox_is_alias"
+
+AlERT_WRONG_MX_RECORD_CUSTOM_DOMAIN = "custom_domain_mx_record_issue"
+
+# alert when a new alias is about to be created on a disabled directory
+ALERT_DIRECTORY_DISABLED_ALIAS_CREATION = "alert_directory_disabled_alias_creation"
+
+ALERT_COMPLAINT_REPLY_PHASE = "alert_complaint_reply_phase"
+ALERT_COMPLAINT_FORWARD_PHASE = "alert_complaint_forward_phase"
+ALERT_COMPLAINT_TRANSACTIONAL_PHASE = "alert_complaint_transactional_phase"
+
+ALERT_QUARANTINE_DMARC = "alert_quarantine_dmarc"
+
+ALERT_DUAL_SUBSCRIPTION_WITH_PARTNER = "alert_dual_sub_with_partner"
+ALERT_WARN_MULTIPLE_SUBSCRIPTIONS = "alert_multiple_subscription"
+
+# <<<<< END ALERT EMAIL >>>>
+
+# Disable onboarding emails
+DISABLE_ONBOARDING = "DISABLE_ONBOARDING" in os.environ
+
+HCAPTCHA_SECRET = os.environ.get("HCAPTCHA_SECRET")
+HCAPTCHA_SITEKEY = os.environ.get("HCAPTCHA_SITEKEY")
+
+PLAUSIBLE_HOST = os.environ.get("PLAUSIBLE_HOST")
+PLAUSIBLE_DOMAIN = os.environ.get("PLAUSIBLE_DOMAIN")
+
+# server host
+HOST = socket.gethostname()
+
+SPAMASSASSIN_HOST = os.environ.get("SPAMASSASSIN_HOST")
+# by default use a tolerant score
+if "MAX_SPAM_SCORE" in os.environ:
+    MAX_SPAM_SCORE = float(os.environ["MAX_SPAM_SCORE"])
+else:
+    MAX_SPAM_SCORE = 5.5
+
+# use a more restrictive score when replying
+if "MAX_REPLY_PHASE_SPAM_SCORE" in os.environ:
+    MAX_REPLY_PHASE_SPAM_SCORE = float(os.environ["MAX_REPLY_PHASE_SPAM_SCORE"])
+else:
+    MAX_REPLY_PHASE_SPAM_SCORE = 5
+
+PGP_SENDER_PRIVATE_KEY = None
+PGP_SENDER_PRIVATE_KEY_PATH = os.environ.get("PGP_SENDER_PRIVATE_KEY_PATH")
+if PGP_SENDER_PRIVATE_KEY_PATH:
+    with open(get_abs_path(PGP_SENDER_PRIVATE_KEY_PATH)) as f:
+        PGP_SENDER_PRIVATE_KEY = f.read()
+
+# the signer address that signs outgoing encrypted emails
+PGP_SIGNER = os.environ.get("PGP_SIGNER")
+
+# emails that have empty From address is sent from this special reverse-alias
+NOREPLY = os.environ.get("NOREPLY", f"noreply@{EMAIL_DOMAIN}")
+
+# list of no reply addresses
+NOREPLIES = sl_getenv("NOREPLIES", list) or [NOREPLY]
+
+COINBASE_WEBHOOK_SECRET = os.environ.get("COINBASE_WEBHOOK_SECRET")
+COINBASE_CHECKOUT_ID = os.environ.get("COINBASE_CHECKOUT_ID")
+COINBASE_API_KEY = os.environ.get("COINBASE_API_KEY")
+try:
+    COINBASE_YEARLY_PRICE = float(os.environ["COINBASE_YEARLY_PRICE"])
+except Exception:
+    COINBASE_YEARLY_PRICE = 30.00
+
+ALIAS_LIMIT = os.environ.get("ALIAS_LIMIT") or "100/day;50/hour;5/minute"
+
+ENABLE_SPAM_ASSASSIN = "ENABLE_SPAM_ASSASSIN" in os.environ
+
+ALIAS_RANDOM_SUFFIX_LENGTH = int(os.environ.get("ALIAS_RAND_SUFFIX_LENGTH", 5))
+
+try:
+    HIBP_SCAN_INTERVAL_DAYS = int(os.environ.get("HIBP_SCAN_INTERVAL_DAYS"))
+except Exception:
+    HIBP_SCAN_INTERVAL_DAYS = 7
+HIBP_API_KEYS = sl_getenv("HIBP_API_KEYS", list) or []
+HIBP_MAX_ALIAS_CHECK = 10_000
+HIBP_RPM = int(os.environ.get("HIBP_API_RPM", 100))
+HIBP_SKIP_PARTNER_ALIAS = os.environ.get("HIBP_SKIP_PARTNER_ALIAS")
+
+KEEP_OLD_DATA_DAYS = 30
+
+POSTMASTER = os.environ.get("POSTMASTER")
+
+# store temporary files, especially for debugging
+TEMP_DIR = os.environ.get("TEMP_DIR")
+
+# Store unsent emails
+SAVE_UNSENT_DIR = os.environ.get("SAVE_UNSENT_DIR")
+if SAVE_UNSENT_DIR and not os.path.isdir(SAVE_UNSENT_DIR):
+    try:
+        os.makedirs(SAVE_UNSENT_DIR)
+    except FileExistsError:
+        pass
+
+# enable the alias automation disable: an alias can be automatically disabled if it has too many bounces
+ALIAS_AUTOMATIC_DISABLE = "ALIAS_AUTOMATIC_DISABLE" in os.environ
+
+# whether the DKIM signing is handled by Rspamd
+RSPAMD_SIGN_DKIM = "RSPAMD_SIGN_DKIM" in os.environ
+
+TWILIO_AUTH_TOKEN = os.environ.get("TWILIO_AUTH_TOKEN")
+
+PHONE_PROVIDER_1_HEADER = "X-SimpleLogin-Secret"
+PHONE_PROVIDER_1_SECRET = os.environ.get("PHONE_PROVIDER_1_SECRET")
+
+PHONE_PROVIDER_2_HEADER = os.environ.get("PHONE_PROVIDER_2_HEADER")
+PHONE_PROVIDER_2_SECRET = os.environ.get("PHONE_PROVIDER_2_SECRET")
+
+ZENDESK_HOST = os.environ.get("ZENDESK_HOST")
+ZENDESK_API_TOKEN = os.environ.get("ZENDESK_API_TOKEN")
+ZENDESK_ENABLED = "ZENDESK_ENABLED" in os.environ
+
+DMARC_CHECK_ENABLED = "DMARC_CHECK_ENABLED" in os.environ
+
+# Bounces can happen after 5 days
+VERP_MESSAGE_LIFETIME = 5 * 86400
+VERP_PREFIX = os.environ.get("VERP_PREFIX") or "sl"
+# Generate with python3 -c 'import secrets; print(secrets.token_hex(28))'
+VERP_EMAIL_SECRET = os.environ.get("VERP_EMAIL_SECRET") or (
+    FLASK_SECRET + "pleasegenerateagoodrandomtoken"
+)
+if len(VERP_EMAIL_SECRET) < 32:
+    raise RuntimeError(
+        "Please, set VERP_EMAIL_SECRET to a random string at least 32 chars long"
+    )
+ALIAS_TRANSFER_TOKEN_SECRET = os.environ.get("ALIAS_TRANSFER_TOKEN_SECRET") or (
+    FLASK_SECRET + "aliastransfertoken"
+)
+
+
+def get_allowed_redirect_domains() -> List[str]:
+    allowed_domains = sl_getenv("ALLOWED_REDIRECT_DOMAINS", list)
+    if allowed_domains:
+        return allowed_domains
+    parsed_url = urlparse(URL)
+    return [parsed_url.hostname]
+
+
+ALLOWED_REDIRECT_DOMAINS = get_allowed_redirect_domains()
+
+
+def setup_nameservers():
+    nameservers = os.environ.get("NAMESERVERS", "1.1.1.1")
+    return nameservers.split(",")
+
+
+NAMESERVERS = setup_nameservers()
+
+DISABLE_CREATE_CONTACTS_FOR_FREE_USERS = os.environ.get(
+    "DISABLE_CREATE_CONTACTS_FOR_FREE_USERS", False
+)
+
+
+# Expect format hits,seconds:hits,seconds...
+# Example 1,10:4,60 means 1 in the last 10 secs or 4 in the last 60 secs
+def getRateLimitFromConfig(
+    env_var: string, default: string = ""
+) -> list[tuple[int, int]]:
+    value = os.environ.get(env_var, default)
+    if not value:
+        return []
+    entries = [entry for entry in value.split(":")]
+    limits = []
+    for entry in entries:
+        fields = entry.split(",")
+        limit = (int(fields[0]), int(fields[1]))
+        limits.append(limit)
+    return limits
+
+
+ALIAS_CREATE_RATE_LIMIT_FREE = getRateLimitFromConfig(
+    "ALIAS_CREATE_RATE_LIMIT_FREE", "10,900:50,3600"
+)
+ALIAS_CREATE_RATE_LIMIT_PAID = getRateLimitFromConfig(
+    "ALIAS_CREATE_RATE_LIMIT_PAID", "50,900:200,3600"
+)
+PARTNER_API_TOKEN_SECRET = os.environ.get("PARTNER_API_TOKEN_SECRET") or (
+    FLASK_SECRET + "partnerapitoken"
+)
+
+JOB_MAX_ATTEMPTS = 5
+JOB_TAKEN_RETRY_WAIT_MINS = 30
+
+# MEM_STORE
+MEM_STORE_URI = os.environ.get("MEM_STORE_URI", None)
+
+# Recovery codes hash salt
+RECOVERY_CODE_HMAC_SECRET = os.environ.get("RECOVERY_CODE_HMAC_SECRET") or (
+    FLASK_SECRET + "generatearandomtoken"
+)
+if not RECOVERY_CODE_HMAC_SECRET or len(RECOVERY_CODE_HMAC_SECRET) < 16:
+    raise RuntimeError(
+        "Please define RECOVERY_CODE_HMAC_SECRET in your configuration with a random string at least 16 chars long"
+    )
+
+
+# the minimum rspamd spam score above which emails that fail DMARC should be quarantined
+if "MIN_RSPAMD_SCORE_FOR_FAILED_DMARC" in os.environ:
+    MIN_RSPAMD_SCORE_FOR_FAILED_DMARC = float(
+        os.environ["MIN_RSPAMD_SCORE_FOR_FAILED_DMARC"]
+    )
+else:
+    MIN_RSPAMD_SCORE_FOR_FAILED_DMARC = None
+
+# run over all reverse alias for an alias and replace them with sender address
+ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT = (
+    "ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT" in os.environ
+)
+
+if ENABLE_ALL_REVERSE_ALIAS_REPLACEMENT:
+    # max number of reverse alias that can be replaced
+    MAX_NB_REVERSE_ALIAS_REPLACEMENT = int(
+        os.environ["MAX_NB_REVERSE_ALIAS_REPLACEMENT"]
+    )
+
+# Only used for tests
+SKIP_MX_LOOKUP_ON_CHECK = False
+
+DISABLE_RATE_LIMIT = "DISABLE_RATE_LIMIT" in os.environ
+
+SUBSCRIPTION_CHANGE_WEBHOOK = os.environ.get("SUBSCRIPTION_CHANGE_WEBHOOK", None)
+MAX_API_KEYS = int(os.environ.get("MAX_API_KEYS", 30))
+
+UPCLOUD_USERNAME = os.environ.get("UPCLOUD_USERNAME", None)
+UPCLOUD_PASSWORD = os.environ.get("UPCLOUD_PASSWORD", None)
+UPCLOUD_DB_ID = os.environ.get("UPCLOUD_DB_ID", None)
+
+STORE_TRANSACTIONAL_EMAILS = "STORE_TRANSACTIONAL_EMAILS" in os.environ
+
+EVENT_WEBHOOK = os.environ.get("EVENT_WEBHOOK", None)
+
+# We want it disabled by default, so only skip if defined
+EVENT_WEBHOOK_SKIP_VERIFY_SSL = "EVENT_WEBHOOK_SKIP_VERIFY_SSL" in os.environ
+EVENT_WEBHOOK_DISABLE = "EVENT_WEBHOOK_DISABLE" in os.environ

app/custom_domain_validation.py

@@ -0,0 +1,37 @@
+from app.db import Session
+from app.dns_utils import get_cname_record
+from app.models import CustomDomain
+
+
+class CustomDomainValidation:
+    def __init__(self, dkim_domain: str):
+        self.dkim_domain = dkim_domain
+        self._dkim_records = {
+            (f"{key}._domainkey", f"{key}._domainkey.{self.dkim_domain}")
+            for key in ("dkim", "dkim02", "dkim03")
+        }
+
+    def get_dkim_records(self) -> {str: str}:
+        """
+        Get a list of dkim records to set up. It will be
+
+        """
+        return self._dkim_records
+
+    def validate_dkim_records(self, custom_domain: CustomDomain) -> dict[str, str]:
+        """
+        Check if dkim records are properly set for this custom domain.
+        Returns empty list if all records are ok. Other-wise return the records that aren't properly configured
+        """
+        invalid_records = {}
+        for prefix, expected_record in self.get_dkim_records():
+            custom_record = f"{prefix}.{custom_domain.domain}"
+            dkim_record = get_cname_record(custom_record)
+            if dkim_record != expected_record:
+                invalid_records[custom_record] = dkim_record or "empty"
+        # HACK: If dkim is enabled, don't disable it to give users time to update their CNAMES
+        if custom_domain.dkim_verified:
+            return invalid_records
+        custom_domain.dkim_verified = len(invalid_records) == 0
+        Session.commit()
+        return invalid_records

app/dashboard/__init__.py

@@ -3,18 +3,71 @@ from .views import (
     pricing,
     setting,
     custom_alias,
+    subdomain,
     billing,
     alias_log,
+    alias_export,
     unsubscribe,
     api_key,
     custom_domain,
     alias_contact_manager,
+    enter_sudo,
     mfa_setup,
     mfa_cancel,
+    fido_setup,
+    coupon,
+    fido_manage,
     domain_detail,
     lifetime_licence,
     directory,
     mailbox,
-    deleted_alias,
     mailbox_detail,
+    refused_email,
+    referral,
+    contact_detail,
+    setup_done,
+    batch_import,
+    alias_transfer,
+    app,
+    delete_account,
+    notification,
+    support,
+    account_setting,
 )
+
+__all__ = [
+    "index",
+    "pricing",
+    "setting",
+    "custom_alias",
+    "subdomain",
+    "billing",
+    "alias_log",
+    "alias_export",
+    "unsubscribe",
+    "api_key",
+    "custom_domain",
+    "alias_contact_manager",
+    "enter_sudo",
+    "mfa_setup",
+    "mfa_cancel",
+    "fido_setup",
+    "coupon",
+    "fido_manage",
+    "domain_detail",
+    "lifetime_licence",
+    "directory",
+    "mailbox",
+    "mailbox_detail",
+    "refused_email",
+    "referral",
+    "contact_detail",
+    "setup_done",
+    "batch_import",
+    "alias_transfer",
+    "app",
+    "delete_account",
+    "notification",
+    "support",
+    "account_setting",
+]

app/dashboard/templates/dashboard/alias_contact_manager.html

@@ -1,114 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "dashboard" %}
-
-{% block title %}
-  Alias Contact Manager
-{% endblock %}
-
-{% block default_content %}
-  <div class="page-header row">
-    <h3 class="page-title col">
-      {{ alias }}
-    </h3>
-  </div>
-
-  <div class="alert alert-primary" role="alert">
-    <p>
-      To send an email from your alias to someone, says <b>friend@example.com</b>, you need to: <br>
-
-      1. Create a special email address called <em>reverse-alias</em> for friend@example.com using the form below <br>
-      2. Send the email to the reverse-alias <em>instead of</em> friend@example.com
-      <br>
-      3. SimpleLogin will send this email from the alias to friend@example.com for you
-    </p>
-    <p>
-      This might sound complicated but trust us, only the first time is a bit awkward.
-    </p>
-    <p>
-      {% if gen_email.mailbox_id %}
-        Make sure you send the email from the mailbox <b>{{ gen_email.mailbox.email }}</b>.
-        This is because only the mailbox that owns the alias can send emails from it.
-      {% else %}
-        Make sure you send the email from your personal email address ({{ current_user.email }}).
-      {% endif %}
-    </p>
-  </div>
-
-  <form method="post">
-    <input type="hidden" name="form-name" value="create">
-    {{ new_contact_form.csrf_token }}
-
-    <label class="form-label">Where do you want to send email to?</label>
-
-    {{ new_contact_form.email(class="form-control", placeholder="First Last <email@example.com>") }}
-    {{ render_field_errors(new_contact_form.email) }}
-    <button class="btn btn-primary mt-2">Create reverse-alias</button>
-  </form>
-
-  <div class="row">
-    {% for forward_email in forward_emails %}
-      <div class="col-md-6">
-        <div class="my-2 p-2 card {% if forward_email.id == forward_email_id %} highlight-row {% endif %}">
-          <div>
-            <span>
-              <a href="{{ 'mailto:' + forward_email.website_send_to() }}"
-                 data-toggle="tooltip"
-                 title="You can click on this to open your email client. Or use the copy button 👉"
-                 class="font-weight-bold">*************************</a>
-
-              <span class="clipboard btn btn-sm btn-success copy-btn" data-toggle="tooltip"
-                    title="Copy to clipboard"
-                    data-clipboard-text="{{ forward_email.website_send_to() }}">
-                Copy reverse-alias
-              </span>
-            </span>
-          </div>
-
-          <div>
-            <i class="fe fe-mail"></i> ➡ {{ forward_email.website_from or forward_email.website_email }}
-          </div>
-
-          <div class="mb-2 text-muted small-text">
-            Created {{ forward_email.created_at | dt }} <br>
-
-            {% if forward_email.last_reply() %}
-              {% set email_log = forward_email.last_reply() %}
-              Last email sent {{ email_log.created_at | dt }}
-            {% endif %}
-          </div>
-
-          <div>
-            <form method="post">
-              <input type="hidden" name="form-name" value="delete">
-              <input type="hidden" name="forward-email-id" value="{{ forward_email.id }}">
-              <span class="card-link btn btn-link float-right delete-forward-email">
-                Delete
-              </span>
-            </form>
-          </div>
-
-        </div>
-      </div>
-    {% endfor %}
-  </div>
-
-{% endblock %}
-
-
-{% block script %}
-  <script>
-    $(".delete-forward-email").on("click", function (e) {
-      notie.confirm({
-        text: "Activity history associated with this reverse-alias will be deleted, " +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/alias_log.html

@@ -1,157 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "dashboard" %}
-{% block head %}
-  <style>
-    {#  https://bootsnipp.com/snippets/rljEW#}
-    .card-counter {
-      box-shadow: 2px 2px 10px #DADADA;
-      margin: 5px;
-      padding: 20px 10px;
-      background-color: #fff;
-      height: 100px;
-      border-radius: 5px;
-      transition: .3s linear all;
-    }
-
-    .card-counter:hover {
-      box-shadow: 4px 4px 20px #DADADA;
-      transition: .3s linear all;
-    }
-
-    .card-counter.primary {
-      background-color: #007bff;
-      color: #FFF;
-    }
-
-    .card-counter.danger {
-      background-color: #ef5350;
-      color: #FFF;
-    }
-
-    .card-counter.success {
-      background-color: #66bb6a;
-      color: #FFF;
-    }
-
-    .card-counter.info {
-      background-color: #26c6da;
-      color: #FFF;
-    }
-
-    .card-counter i {
-      font-size: 5em;
-      opacity: 0.2;
-    }
-
-    .card-counter .count-numbers {
-      position: absolute;
-      right: 35px;
-      top: 20px;
-      font-size: 32px;
-      display: block;
-    }
-
-    .card-counter .count-name {
-      position: absolute;
-      right: 35px;
-      top: 65px;
-      text-transform: capitalize;
-      opacity: 0.5;
-      display: block;
-      font-size: 18px;
-    }
-  </style>
-{% endblock %}
-{% block title %}
-  Alias Activity
-{% endblock %}
-
-{% block default_content %}
-  <div class="page-header row ml-0">
-    <h3 class="page-title col">
-      {{ alias }}
-    </h3>
-  </div>
-  <div class="row">
-    <div class="col-md-3 col-sm-6">
-      <div class="card-counter primary">
-        <i class="fa fa-at"></i>
-        <span class="count-numbers">{{ total }}</span>
-        <span class="count-name">Email Handled</span>
-      </div>
-    </div>
-    <div class="col-md-3 col-sm-6">
-      <div class="card-counter primary">
-        <i class="fa fa-paper-plane"></i>
-        <span class="count-numbers">{{ email_forwarded }}</span>
-        <span class="count-name">Email Forwarded</span>
-      </div>
-    </div>
-    <div class="col-md-3 col-sm-6">
-      <div class="card-counter primary">
-        <i class="fa fa-reply"></i>
-        <span class="count-numbers">{{ email_replied }}</span>
-        <span class="count-name">Email Replied</span>
-      </div>
-    </div>
-    <div class="col-md-3 col-sm-6">
-      <div class="card-counter danger">
-        <i class="fa fa-ban"></i>
-        <span class="count-numbers">{{ email_blocked }}</span>
-        <span class="count-name">Email Blocked</span>
-      </div>
-    </div>
-  </div>
-  <h2 class="pt-4">Activities</h2>
-  <div class="row">
-    {% for log in logs %}
-      <div class="col-12">
-        <div class="my-2 p-2 card border-light">
-          <div class="font-weight-bold">{{ log.when | dt }}
-            {% if log.bounced %}  ⚠️ {% endif %}
-          </div>
-
-          <div>
-            {% if log.bounced %}
-              <span class="mr-2">{{ log.website_from or log.website_email }}</span>
-              <img src="{{ url_for('static', filename='arrows/forward-arrow.svg') }}" class="arrow">
-              <span class="ml-2">{{ log.alias }}</span>
-              <img src="{{ url_for('static', filename='arrows/blocked-arrow.svg') }}" class="arrow">
-              <span class="ml-2">{{ log.mailbox }}</span>
-            {% else %}
-              <span class="mr-2">{{ log.website_from or log.website_email }}</span>
-
-              <span>
-              {% if log.is_reply %}
-                <img src="{{ url_for('static', filename='arrows/reply-arrow.svg') }}" class="arrow">
-              {% elif log.blocked %}
-                <img src="{{ url_for('static', filename='arrows/blocked-arrow.svg') }}" class="arrow">
-              {% else %}
-                <img src="{{ url_for('static', filename='arrows/forward-arrow.svg') }}" class="arrow">
-              {% endif %}
-            </span>
-
-              <span class="ml-2">{{ log.alias }}</span>
-            {% endif %}
-          </div>
-        </div>
-      </div>
-    {% endfor %}
-  </div>
-
-  <nav aria-label="Alias log navigation">
-    <ul class="pagination">
-      <li class="page-item {% if page_id == 0 %}disabled{% endif %}">
-        <a class="page-link"
-           href="{{ url_for('dashboard.alias_log', alias_id=alias_id, page_id=page_id-1) }}">Previous</a>
-      </li>
-      <li class="page-item {% if last_page %}disabled{% endif %}">
-        <a class="page-link" href="{{ url_for('dashboard.alias_log', alias_id=alias_id, page_id=page_id+1) }}">Next</a>
-      </li>
-    </ul>
-  </nav>
-{% endblock %}
-
-{% block script %}
-{% endblock %}

app/dashboard/templates/dashboard/api_key.html

@@ -1,137 +0,0 @@
-{% extends 'default.html' %}
-
-{% block title %}
-  API Key
-{% endblock %}
-
-{% set active_page = "api_key" %}
-
-{% block head %}
-{% endblock %}
-
-{% block default_content %}
-  <div class="row">
-    <div class="col-md-8 offset-md-2">
-      <h1 class="h3"> API Key </h1>
-
-      <div class="alert alert-primary" role="alert">
-        The API Key is used on the SimpleLogin Chrome/Firefox/Safari extension. <br>
-        You can install the Chrome extension on
-        <a href="https://chrome.google.com/webstore/detail/simplelogin-extension/dphilobhebphkdjbpfohgikllaljmgbn"
-           target="_blank">Chrome Store<i class="fe fe-external-link"></i></a>,
-        Firefox add-on on <a href="https://addons.mozilla.org/en-GB/firefox/addon/simplelogin/"
-                             target="_blank">Firefox<i
-          class="fe fe-external-link"></i></a>
-        and Safari extension on <a
-          href="https://apps.apple.com/us/app/simplelogin/id1494051017?mt=12&fbclid=IwAR0M0nnEKgoieMkmx91TSXrtcScj7GouqRxGgXeJz2un_5ydhIKlbAI79Io"
-          target="_blank">AppStore<i class="fe fe-external-link"></i></a>
-        <br>
-        Please copy and paste the API key below into the extension to get started. <br>
-        <span class="text-danger">
-          ⚠️Your API Keys are secret and should be treated as passwords.
-        </span>
-      </div>
-
-      {% for api_key in api_keys %}
-        <div class="card" style="max-width: 50rem">
-          <div class="card-body">
-            <h5 class="card-title">{{ api_key.name }}</h5>
-            <h6 class="card-subtitle mb-2 text-muted">
-              {% if api_key.last_used %}
-                Last used: {{ api_key.last_used | dt }} <br>
-                Used: {{ api_key.times }} times.
-              {% else %}
-                Never used
-              {% endif %}
-            </h6>
-
-            <div class="input-group">
-              <input class="form-control" id="apikey-{{ api_key.id }}" readonly value="**********">
-              <div class="input-group-append">
-                <span class="input-group-text">
-                  <i class="fe fe-eye toggle-api-key" data-show="off" data-secret="{{ api_key.code }}"
-                  ></i>
-                </span>
-              </div>
-            </div>
-
-            <br>
-
-            <div class="row">
-              <div class="col">
-                <button class="clipboard btn btn-primary" data-clipboard-action="copy"
-                        data-clipboard-text="{{ api_key.code }}"
-                        data-clipboard-target="#apikey-{{ api_key.id }}">
-                  Copy &nbsp; &nbsp; <i class="fe fe-clipboard"></i>
-                </button>
-              </div>
-
-              <div class="col">
-                <form method="post">
-                  <input type="hidden" name="form-name" value="delete">
-                  <input type="hidden" name="api-key-id" value="{{ api_key.id }}">
-                  <span class="card-link btn btn-link float-right delete-api-key">
-                    Delete
-                  </span>
-                </form>
-              </div>
-            </div>
-
-          </div>
-        </div>
-      {% endfor %}
-
-      <hr>
-
-      <form method="post">
-        {{ new_api_key_form.csrf_token }}
-        <input type="hidden" name="form-name" value="create">
-
-        <label class="form-label">Api Key Name</label>
-        <small>Name of the api key, e.g. where it will be used.</small>
-
-        {{ new_api_key_form.name(class="form-control", placeholder="Chrome, Firefox") }}
-        {{ render_field_errors(new_api_key_form.name) }}
-        <button class="btn btn-lg btn-success mt-2">Create</button>
-      </form>
-
-
-    </div>
-
-  </div>
-{% endblock %}
-
-{% block script %}
-  <script>
-    $(".delete-api-key").on("click", function (e) {
-      notie.confirm({
-        text: "If this api key is currently in use, you need to replace it with another api key, " +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-
-    $(".toggle-api-key").on('click', function (event) {
-      let that = $(this);
-      let apiInput = that.parent().parent().parent().find("input");
-      if (that.attr("data-show") === "off") {
-        let apiKey = $(this).attr("data-secret");
-        apiInput.val(apiKey);
-        that.addClass("fe-eye-off");
-        that.removeClass("fe-eye");
-        that.attr("data-show", "on");
-      } else {
-        that.removeClass("fe-eye-off");
-        that.addClass("fe-eye");
-        apiInput.val("**********");
-        that.attr("data-show", "off");
-      }
-
-    });
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/billing.html

@@ -1,79 +0,0 @@
-{% extends 'default.html' %}
-
-{% block title %}
-  Billing
-{% endblock %}
-
-{% block head %}
-{% endblock %}
-
-{% block default_content %}
-  <div class="bg-white p-6" style="max-width: 60em; margin: auto">
-    <h1 class="h3 mb-5"> Billing </h1>
-
-    {% if sub.cancelled %}
-      <p>
-        You are on the <b>{{ sub.plan_name() }}</b> plan. <br>
-        You have canceled your subscription and it will end on {{current_user.next_bill_date()}}
-        ({{ sub.next_bill_date | dt }}).
-      </p>
-
-      <hr>
-      <p>
-        If you change your mind you can subscribe again to SimpleLogin but please note that this will be a completely
-        new subscription and
-        your payment method will be charged <b>immediately</b>.
-        <br>
-
-        We are going to send you an email by the end of the subscription so maybe you can upgrade at that time.
-        <br>
-        <a href="{{ url_for('dashboard.pricing') }}" class="btn btn-primary mt-2">Re-subscribe</a>
-      </p>
-
-    {% else %}
-      <p>
-        You are on the <b>{{ sub.plan_name() }}</b> plan. Thank you very much for supporting
-        SimpleLogin. 🙌
-      </p>
-
-      <div class="mt-3">
-        Click here to update billing information on Paddle, our payment partner: <br>
-        <a class="btn btn-success" href="{{ sub.update_url }}"> Update billing information </a>
-      </div>
-
-      <hr>
-
-      <div>
-        Don't want to protect your inbox anymore? <br>
-
-        <form method="post">
-          <input type="hidden" name="form-name" value="cancel">
-
-          <span class="cancel btn btn-warning">
-            Cancel subscription <i class="fe fe-alert-triangle text-danger"></i>
-          </span>
-        </form>
-
-      </div>
-    {% endif %}
-  </div>
-
-{% endblock %}
-
-{% block script %}
-  <script>
-
-    $(".cancel").on("click", function (e) {
-      notie.confirm({
-        text: `This operation is irreversible, please confirm`,
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/custom_alias.html

@@ -1,91 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "dashboard" %}
-
-{% block title %}
-  Custom Alias
-{% endblock %}
-
-{% block default_content %}
-
-  <div class="bg-white p-6" style="max-width: 60em; margin: auto">
-    <h1 class="h3 mb-5">New Email Alias</h1>
-
-    {% if  user_custom_domains|length == 0 and not DISABLE_ALIAS_SUFFIX %}
-      <div class="row">
-        <div class="col p-1">
-          <div class="alert alert-primary" role="alert">
-            You might notice a random word after the dot(<em>.</em>) in the alias.
-            This part is to avoid a person taking all the "nice" aliases like <b>hello@{{ EMAIL_DOMAIN }}</b>,
-            <b>me@{{ EMAIL_DOMAIN }}</b>, etc. <br>
-            If you add your own domain, this restriction is removed and you can fully customize the alias. <br>
-          </div>
-        </div>
-      </div>
-    {% endif %}
-
-    <form method="post">
-      <div class="row mb-2">
-        <div class="col-sm-6 mb-1 p-1" style="min-width: 4em">
-          <input name="prefix" class="form-control"
-                 type="text"
-                 pattern="[0-9a-z-_]{1,}"
-                 title="Only lowercase letter, number, dash (-), underscore (_) can be used in alias prefix."
-                 placeholder="email alias, for example newsletter-123_xyz"
-                 autofocus required>
-          <div class="small-text">
-            Only lowercase letter, number, dash (-), underscore (_) can be used.
-          </div>
-        </div>
-
-
-        <div class="col-sm-6 p-1">
-          <select class="form-control custom-select" name="suffix">
-            {% for suffix in suffixes %}
-              <option value="{{ suffix[1] }}">
-                {% if suffix[0] %}
-                  {{ suffix[1] }} (your domain)
-                {% else %}
-                  {{ suffix[1] }}
-                {% endif %}
-              </option>
-            {% endfor %}
-          </select>
-        </div>
-      </div>
-
-      <div class="row mb-2">
-        <div class="col p-1">
-          <select class="form-control custom-select" name="mailbox">
-            {% for mailbox in mailboxes %}
-              <option value="{{ mailbox }}">
-                {{ mailbox }}
-              </option>
-            {% endfor %}
-          </select>
-          <div class="small-text">
-            The mailbox that owns this alias.
-          </div>
-        </div>
-      </div>
-
-      <div class="row mb-2">
-        <div class="col p-1">
-        <textarea name="note"
-                  class="form-control"
-                  rows="3"
-                  placeholder="Note, can be anything to help you remember WHY you create this alias. This field is optional."></textarea>
-        </div>
-      </div>
-
-
-      <div class="row">
-        <div class="col p-1">
-          <button class="btn btn-primary mt-1">Create</button>
-        </div>
-      </div>
-    </form>
-  </div>
-
-{% endblock %}
-

app/dashboard/templates/dashboard/custom_domain.html

@@ -1,88 +0,0 @@
-{% extends 'default.html' %}
-{% set active_page = "custom_domain" %}
-
-{% block title %}
-  Custom Domains
-{% endblock %}
-
-{% block head %}
-{% endblock %}
-
-{% block default_content %}
-  <div class="row">
-    <div class="col-md-8 offset-md-2">
-      <h1 class="h3"> Custom Domains </h1>
-
-      {% if not current_user.is_premium() %}
-        <div class="alert alert-danger" role="alert">
-          This feature is only available in premium plan.
-        </div>
-      {% endif %}
-
-      <div class="alert alert-primary" role="alert">
-        If you own a domain, let's say <b>example.com</b>, you will be able to create aliases with this domain, for example
-        contact@example.com, help@example.com, etc with SimpleLogin. <br>
-        You could also enable <b>catch-all</b> feature that allows you to create aliases on-the-fly.
-      </div>
-
-      {% for custom_domain in custom_domains %}
-        <div class="card" style="max-width: 50rem">
-          <div class="card-body">
-            <h5 class="card-title">
-              <a href="{{ url_for('dashboard.domain_detail', custom_domain_id=custom_domain.id) }}">{{ custom_domain.domain }}</a>
-              {% if custom_domain.verified %}
-                <span class="cursor" data-toggle="tooltip" data-original-title="Domain Verified">✅</span>
-              {% else %}
-                <span class="cursor" data-toggle="tooltip" data-original-title="DNS Setup Needed">
-                  <a href="{{ url_for('dashboard.domain_detail_dns', custom_domain_id=custom_domain.id) }}"
-                     class="text-decoration-none">🚫
-                  </a>
-                </span>
-              {% endif %}
-            </h5>
-            <h6 class="card-subtitle mb-2 text-muted">
-              Created {{ custom_domain.created_at | dt }} <br>
-              <span class="font-weight-bold">{{ custom_domain.nb_alias() }}</span> aliases.
-            </h6>
-
-            <a href="{{ url_for('dashboard.domain_detail', custom_domain_id=custom_domain.id) }}">Details ➡</a>
-          </div>
-        </div>
-      {% endfor %}
-
-      <hr>
-
-      <form method="post">
-        {{ new_custom_domain_form.csrf_token }}
-        <input type="hidden" name="form-name" value="create">
-
-        <label class="form-label">Domain</label>
-        <small>Please use full path domain, for ex <em>my-subdomain.my-domain.com</em></small>
-
-        {{ new_custom_domain_form.domain(class="form-control", placeholder="my-domain.com") }}
-        {{ render_field_errors(new_custom_domain_form.domain) }}
-        <button class="btn btn-lg btn-success mt-2">Create</button>
-      </form>
-
-
-    </div>
-
-  </div>
-{% endblock %}
-
-{% block script %}
-  <script>
-    $(".delete-custom-domain").on("click", function (e) {
-      notie.confirm({
-        text: "All aliases associated with this domain will be also deleted, " +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/deleted_alias.html

@@ -1,31 +0,0 @@
-{% extends 'default.html' %}
-
-{% block title %}
-  Deleted Aliases
-{% endblock %}
-
-{% block head %}
-{% endblock %}
-
-{% block default_content %}
-  <div style="max-width: 60em; margin: auto">
-    <h1 class="h3 mb-5"> Deleted Aliases </h1>
-
-    {% if deleted_aliases|length == 0 %}
-      <div class="my-4 p-4 card">
-        You haven't deleted any alias.
-      </div>
-    {% endif %}
-
-    {% for deleted_alias in deleted_aliases %}
-      <div class="my-4 p-4 card border-light">
-        {{ deleted_alias.email }}
-        <div class="small-text">
-          Deleted {{ deleted_alias.created_at | dt }}
-        </div>
-      </div>
-    {% endfor %}
-
-
-  </div>
-{% endblock %}

app/dashboard/templates/dashboard/directory.html

@@ -1,114 +0,0 @@
-{% extends 'default.html' %}
-{% set active_page = "directory" %}
-
-{% block title %}
-  Directory
-{% endblock %}
-
-{% block default_content %}
-  <div class="row">
-    <div class="col-md-8 offset-md-2">
-      <h1 class="h3"> Directories </h1>
-
-      {% if not current_user.is_premium() %}
-        <div class="alert alert-danger" role="alert">
-          This feature is only available in premium plan.
-        </div>
-      {% endif %}
-
-      <div class="alert alert-primary" role="alert">
-        Directory allows you to create aliases <b>on the fly</b>. Simply use <br>
-        <div class="pl-3 py-2 bg-white">
-          <em>your_directory/<b>anything</b>@{{ EMAIL_DOMAIN }}</em> or <br>
-          <em>your_directory+<b>anything</b>@{{ EMAIL_DOMAIN }}</em> or <br>
-          <em>your_directory#<b>anything</b>@{{ EMAIL_DOMAIN }}</em> <br>
-        </div>
-        next time you need an email address. <br>
-        <em><b>anything</b></em> could really be anything, it's up to you to invent the most creative alias 😉. <br>
-        <em>your_directory</em> is the name of one of your directories. <br><br>
-        You can use the directory feature on the following domains:
-        {% for alias_domain in ALIAS_DOMAINS %}
-          <div class="font-weight-bold">{{ alias_domain }} </div>
-        {% endfor %}
-
-        <div class="h4 text-primary mt-3">
-          ℹ️
-          The alias will be created the first time it receives an email.
-        </div>
-
-      </div>
-
-      {% for dir in dirs %}
-        <div class="card" style="max-width: 50rem">
-          <div class="card-body">
-            <h5 class="card-title">
-              {{ dir.name }}
-            </h5>
-            <h6 class="card-subtitle mb-2 text-muted">
-              Created {{ dir.created_at | dt }} <br>
-              <span class="font-weight-bold">{{ dir.nb_alias() }}</span> aliases.
-            </h6>
-          </div>
-
-          <div class="card-footer p-0">
-            <div class="row">
-              <div class="col">
-                <form method="post">
-                  <input type="hidden" name="form-name" value="delete">
-                  <input type="hidden" class="dir-name" value="{{ dir.name }}">
-                  <input type="hidden" name="dir-id" value="{{ dir.id }}">
-                  <span class="card-link btn btn-link float-right text-danger delete-dir">
-                    Delete
-                  </span>
-                </form>
-              </div>
-            </div>
-
-          </div>
-
-        </div>
-      {% endfor %}
-
-      {% if dirs|length > 0 %}
-        <hr>
-      {% endif %}
-
-      <form method="post" class="mt-6">
-        {{ new_dir_form.csrf_token }}
-        <input type="hidden" name="form-name" value="create">
-
-        <div class="font-weight-bold">Directory Name</div>
-        <div class="small-text">
-          Directory name must be at least 3 characters.
-          Only lowercase letter, number, dash (-), underscore (_) can be used.
-        </div>
-
-        {{ new_dir_form.name(class="form-control", placeholder="my-directory",
-            pattern="[0-9a-z-_]{3,}",
-            title="Only letter, number, dash (-), underscore (_) can be used. Directory name must be at least 3 characters.") }}
-        {{ render_field_errors(new_dir_form.name) }}
-        <button class="btn btn-lg btn-success mt-2">Create</button>
-      </form>
-
-    </div>
-
-  </div>
-{% endblock %}
-
-{% block script %}
-  <script>
-    $(".delete-dir").on("click", function (e) {
-      let directory = $(this).parent().find(".dir-name").val();
-      notie.confirm({
-        text: `All aliases associated with <b>${directory}</b> directory will be also deleted, ` +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/domain_detail/base.html

@@ -1,35 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "custom_domain" %}
-
-{% block default_content %}
-  <div class="row">
-    <div class="col-lg-3 order-lg-1 mb-4">
-      <div class="list-group list-group-transparent mb-0">
-        <a href="{{ url_for('dashboard.domain_detail', custom_domain_id=custom_domain.id) }}"
-           class="list-group-item list-group-item-action {{ 'active' if domain_detail_page == 'info' }}">
-          <span class="icon mr-3"><i class="fe fe-flag"></i></span>Info
-        </a>
-
-        <a href="{{ url_for('dashboard.domain_detail_dns', custom_domain_id=custom_domain.id) }}"
-           class="list-group-item list-group-item-action {{ 'active' if domain_detail_page == 'dns' }}">
-          <span class="icon mr-3"><i class="fe fe-cloud"></i></span>DNS
-        </a>
-      </div>
-    </div>
-
-    <div class="col-lg-9">
-      <div class="card">
-        <div class="card-body">
-          <div class="text-wrap p-lg-6">
-            {% block domain_detail_content %}
-            {% endblock %}
-          </div>
-        </div>
-      </div>
-    </div>
-  </div>
-
-{% endblock %}
-
-

app/dashboard/templates/dashboard/domain_detail/dns.html

@@ -1,198 +0,0 @@
-{% extends 'dashboard/domain_detail/base.html' %}
-
-{% set domain_detail_page = "dns" %}
-
-{% block title %}
-  {{ custom_domain.domain }} DNS
-{% endblock %}
-
-
-{% block domain_detail_content %}
-  <div class="bg-white p-4" style="max-width: 60rem; margin: auto">
-    <h1 class="h3"> {{ custom_domain.domain }} </h1>
-    <div class="">Please follow the steps below to set up your domain.</div>
-
-    <div class="small-text mb-5">
-      DNS changes could take up to 24 hours to propagate. In practice, it's a lot faster though (~1
-      minute or in our experience).
-    </div>
-
-    <div id="mx-form">
-      <div class="font-weight-bold">1. MX record
-
-        {% if custom_domain.verified %}
-          <span class="cursor" data-toggle="tooltip" data-original-title="MX Record Verified">✅</span>
-        {% else %}
-          <span class="cursor" data-toggle="tooltip" data-original-title="MX Record Not Verified">🚫 </span>
-        {% endif %}
-      </div>
-
-      <div class="mb-2">Add the following MX DNS record to your domain. <br>
-        Please note that there's a point (<em>.</em>) at the end target addresses. <br>
-        Also some domain registrars (Namecheap, CloudFlare, etc) might use <em>@</em> for the root domain.
-      </div>
-
-      {% for priority, email_server in EMAIL_SERVERS_WITH_PRIORITY %}
-        <div class="mb-3 p-3" style="background-color: #eee">
-          Domain: <em>{{ custom_domain.domain }}</em> or <em>@</em> <br>
-          Priority: {{ priority }} <br>
-          Target: <em>{{ email_server }}</em> <br>
-        </div>
-      {% endfor %}
-
-      <form method="post" action="#mx-form">
-        <input type="hidden" name="form-name" value="check-mx">
-        {% if custom_domain.verified %}
-          <button type="submit" class="btn btn-outline-primary">
-            Re-verify
-          </button>
-        {% else %}
-          <button type="submit" class="btn btn-primary">
-            Verify
-          </button>
-        {% endif %}
-      </form>
-
-      {% if not mx_ok %}
-        <div class="text-danger mt-4">
-          Your DNS is not correctly set. The MX record we obtain is:
-          <div class="mb-3 p-3" style="background-color: #eee">
-            {% if not mx_errors %}
-              (Empty)
-            {% endif %}
-            {% for r in mx_errors %}
-              {{ r }} <br>
-            {% endfor %}
-          </div>
-          {% if custom_domain.verified %}
-            Please make sure to fix this ASAP - your aliases might not work properly.
-          {% endif %}
-        </div>
-      {% endif %}
-    </div>
-
-    <hr>
-
-    <div id="spf-form">
-      <div class="font-weight-bold">2. SPF (Optional)
-        {% if custom_domain.spf_verified %}
-          <span class="cursor" data-toggle="tooltip" data-original-title="SPF Verified">✅</span>
-        {% else %}
-          <span class="cursor" data-toggle="tooltip" data-original-title="SPF Not Verified">🚫 </span>
-        {% endif %}
-      </div>
-
-      <div>
-        SPF <a href="https://en.wikipedia.org/wiki/Sender_Policy_Framework" target="_blank">(Wikipedia↗)</a> is an email
-        authentication method
-        designed to detect forging sender addresses during the delivery of the email. <br>
-        Setting up SPF is highly recommended to reduce the chance your emails ending up in the recipient's Spam folder.
-      </div>
-
-      <div class="mb-2">Add the following TXT DNS record to your domain</div>
-
-      <div class="mb-2 p-3" style="background-color: #eee">
-        Domain: <em>{{ custom_domain.domain }}</em> or <em>@</em> <br>
-        Value:
-        <em>
-          {{ spf_record }}
-        </em>
-      </div>
-
-      <form method="post" action="#spf-form">
-        <input type="hidden" name="form-name" value="check-spf">
-        {% if custom_domain.spf_verified %}
-          <button type="submit" class="btn btn-outline-primary">
-            Re-verify
-          </button>
-        {% else %}
-          <button type="submit" class="btn btn-primary">
-            Verify
-          </button>
-        {% endif %}
-      </form>
-
-      {% if not spf_ok %}
-        <div class="text-danger mt-4">
-          Your DNS is not correctly set. The TXT record we obtain is:
-          <div class="mb-3 p-3" style="background-color: #eee">
-            {% if not spf_errors %}
-              (Empty)
-            {% endif %}
-
-            {% for r in spf_errors %}
-              {{ r }} <br>
-            {% endfor %}
-          </div>
-          {% if custom_domain.spf_verified %}
-            Without SPF setup, emails you sent from your alias might end up in Spam/Junk folder.
-          {% endif %}
-        </div>
-      {% endif %}
-    </div>
-
-    <hr>
-
-    <div id="dkim-form">
-      <div class="font-weight-bold">3. DKIM (Optional)
-        {% if custom_domain.dkim_verified %}
-          <span class="cursor" data-toggle="tooltip" data-original-title="SPF Verified">✅</span>
-        {% else %}
-          <span class="cursor" data-toggle="tooltip" data-original-title="DKIM Not Verified">🚫 </span>
-        {% endif %}
-      </div>
-
-      <div>
-        DKIM <a href="https://en.wikipedia.org/wiki/DomainKeys_Identified_Mail" target="_blank">(Wikipedia↗)</a> is an
-        email
-        authentication method
-        designed to avoid email spoofing. <br>
-        Setting up DKIM is highly recommended to reduce the chance your emails ending up in the recipient's Spam folder.
-      </div>
-
-      <div class="mb-2">Add the following TXT DNS record to your domain</div>
-
-      <div class="mb-2 p-3" style="background-color: #eee">
-        Domain: <em>dkim._domainkey.{{ custom_domain.domain }}</em> <br>
-        Value:
-        <em style="overflow-wrap: break-word">
-          {{ dkim_record }}
-        </em>
-      </div>
-
-      <form method="post" action="#dkim-form">
-        <input type="hidden" name="form-name" value="check-dkim">
-        {% if custom_domain.dkim_verified %}
-          <button type="submit" class="btn btn-outline-primary">
-            Re-verify
-          </button>
-        {% else %}
-          <button type="submit" class="btn btn-primary">
-            Verify
-          </button>
-        {% endif %}
-      </form>
-
-      {% if not dkim_ok %}
-        <div class="text-danger mt-4">
-          Your DNS is not correctly set.
-          {% if dkim_errors %}
-            The TXT record we obtain for
-            <em>dkim._domainkey.{{ custom_domain.domain }}</em> is:
-
-            <div class="mb-3 p-3" style="background-color: #eee">
-              {% for r in dkim_errors %}
-                {{ r }} <br>
-              {% endfor %}
-            </div>
-          {% endif %}
-
-          {% if custom_domain.dkim_verified %}
-            Without DKIM setup, emails you sent from your alias might end up in Spam/Junk folder.
-          {% endif %}
-        </div>
-      {% endif %}
-    </div>
-
-  </div>
-{% endblock %}

app/dashboard/templates/dashboard/domain_detail/info.html

@@ -1,86 +0,0 @@
-{% extends 'dashboard/domain_detail/base.html' %}
-
-{% set domain_detail_page = "info" %}
-
-{% block title %}
-  {{ custom_domain.domain }} Info
-{% endblock %}
-
-{% block domain_detail_content %}
-  <h1 class="h3"> {{ custom_domain.domain }}
-    {% if custom_domain.verified %}
-      <span class="cursor" data-toggle="tooltip" data-original-title="DNS Setup OK">✅</span>
-    {% else %}
-      <span class="cursor" data-toggle="tooltip" data-original-title="DNS Setup Needed">
-        <a href="{{ url_for('dashboard.domain_detail_dns', custom_domain_id=custom_domain.id) }}"
-           class="text-decoration-none">🚫
-        </a>
-      </span>
-    {% endif %}
-  </h1>
-
-  <div class="small-text">Created {{ custom_domain.created_at | dt }}</div>
-
-  {{ nb_alias }} aliases
-
-  <hr>
-  <div>Catch All</div>
-  <div class="small-text">
-    This feature allows you to create aliases <b>on the fly</b>.
-    Simply use <em>anything@{{ custom_domain.domain }}</em>
-    next time you need an email address. <br>
-    The alias will be created the first time it receives an email.
-  </div>
-
-  <div>
-    <form method="post">
-      <input type="hidden" name="form-name" value="switch-catch-all">
-      <label class="custom-switch cursor mt-2 pl-0"
-             data-toggle="tooltip"
-          {% if custom_domain.catch_all %}
-             title="Disable catch-all"
-          {% else %}
-             title="Enable catch-all"
-          {% endif %}
-      >
-        <input type="checkbox" class="custom-switch-input"
-            {{ "checked" if custom_domain.catch_all else "" }}>
-
-        <span class="custom-switch-indicator"></span>
-      </label>
-    </form>
-  </div>
-
-  <hr>
-  <h3 class="mb-0">Delete Domain</h3>
-  <div class="small-text mb-3">Please note that this operation is irreversible.
-    All aliases associated with this domain will be also deleted
-  </div>
-
-  <form method="post">
-    <input type="hidden" name="form-name" value="delete">
-    <span class="delete-custom-domain btn btn-outline-danger">Delete domain</span>
-  </form>
-
-{% endblock %}
-
-{% block script %}
-  <script>
-    $(".custom-switch-input").change(function (e) {
-      $(this).closest("form").submit();
-    });
-
-    $(".delete-custom-domain").on("click", function (e) {
-      notie.confirm({
-        text: "All aliases associated with <b>{{ custom_domain.domain }}</b> will be also deleted, " +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/index.html

@@ -1,383 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "dashboard" %}
-
-{% block head %}
-  <style>
-    .alias-activity {
-      font-weight: 600;
-      font-size: 14px;
-    }
-
-    .btn-group-border-left {
-      border-left: 1px #fbfbfb4f solid;
-    }
-  </style>
-{% endblock %}
-
-{% block title %}
-  Alias
-{% endblock %}
-
-{% block default_content %}
-  <div class="page-header row" style="margin-top: 0rem">
-    <div class="col-lg-3 col-sm-12 p-0 mt-1">
-      <form method="get">
-        <input type="search" name="query" autofocus placeholder="Enter to search for alias" class="form-control shadow"
-               value="{{ query }}">
-      </form>
-    </div>
-
-    <div class="col-lg-5 offset-lg-4 pr-0 mt-1">
-      <div class="btn-group float-right" role="group">
-        <form method="post">
-          <input type="hidden" name="form-name" value="create-custom-email">
-          <button data-toggle="tooltip"
-                  title="Create a custom alias"
-                  class="btn btn-primary mr-2"><i class="fa fa-plus"></i> New Email Alias
-          </button>
-        </form>
-        <div class="btn-group" role="group">
-          <form method="post">
-            <input type="hidden" name="form-name" value="create-random-email">
-            <button data-toggle="tooltip"
-                    title="Create a totally random alias"
-                    class="btn btn-success"><i class="fa fa-random"></i> Random Alias
-            </button>
-          </form>
-          <button id="btnGroupDrop1" type="button" class="btn btn-success dropdown-toggle btn-group-border-left"
-                  data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-          </button>
-          <div class="dropdown-menu dropdown-menu-right border-left" aria-labelledby="btnGroupDrop1">
-            <div class="">
-              <form method="post">
-                <input type="hidden" name="form-name" value="create-random-email">
-                <input type="hidden" name="generator_scheme" value="{{ AliasGeneratorEnum.word.value }}">
-                <button class="dropdown-item">By Random Words</button>
-              </form>
-            </div>
-            <div class="">
-              <form method="post">
-                <input type="hidden" name="form-name" value="create-random-email">
-                <input type="hidden" name="generator_scheme" value="{{ AliasGeneratorEnum.uuid.value }}">
-                <button class="dropdown-item">By UUID</button>
-              </form>
-            </div>
-          </div>
-        </div>
-      </div>
-    </div>
-
-  </div>
-
-  <div class="row">
-    {% for alias_info in aliases %}
-      {% set gen_email = alias_info.gen_email %}
-
-      <div class="col-12 col-lg-6">
-        <div class="card p-4 shadow-sm {% if alias_info.highlight %} highlight-row {% endif %} ">
-          <div class="row">
-            <div class="col-8">
-            <span class="clipboard cursor mb-0"
-                {% if loop.index ==1 %}
-                  data-intro="This is an <em>alias</em>. <br><br>
-        <b>All</b> emails sent to an alias will be <em>forwarded</em> to your inbox. <br><br>
-        Alias is a great way to hide your personal email address so feel free to
-        use it whenever possible, for example when signing up for a newsletter or creating a new account on a suspicious website 😎"
-                  data-step="2"
-                {% endif %}
-                {% if gen_email.enabled %}
-                  data-toggle="tooltip"
-                  title="Copy to clipboard"
-                  data-clipboard-text="{{ gen_email.email }}"
-                {% endif %}
-            >
-              <span class="font-weight-bold">{{ gen_email.email }}</span>
-              {% if gen_email.enabled %}
-                <span class="btn btn-sm btn-success copy-btn">
-                  Copy
-                </span>
-              {% endif %}
-            </span>
-            </div>
-            <div class="col text-right">
-              <form method="post">
-                <input type="hidden" name="form-name" value="switch-email-forwarding">
-                <input type="hidden" name="gen-email-id" value="{{ gen_email.id }}">
-                <label class="custom-switch cursor"
-                       data-toggle="tooltip"
-                    {% if gen_email.enabled %}
-                       title="Block Alias"
-                    {% else %}
-                       title="Unblock Alias"
-                    {% endif %}
-
-                    {% if loop.index ==1 %}
-                       data-intro="By turning off an alias, emails sent to this alias will <em>not</em>
-                           be forwarded to your inbox. <br><br>
-                           This should be used with care as others might
-                           not be able to reach you after ...
-                            "
-                       data-step="3"
-                    {% endif %}
-                       style="padding-left: 0px"
-                >
-                  <input type="hidden" name="alias" class="alias" value="{{ gen_email.email }}">
-                  <input type="checkbox" class="custom-switch-input"
-                      {{ "checked" if gen_email.enabled else "" }}>
-
-                  <span class="custom-switch-indicator"></span>
-                </label>
-              </form>
-            </div>
-          </div>
-
-          <hr class="my-2">
-
-          <p class="small-text">
-            Created {{ gen_email.created_at | dt }}
-            {% if alias_info.highlight %}
-              - <span class="font-weight-bold text-success small-text">New</span>
-            {% endif %}
-          </p>
-
-          <div class="" style="font-size: 12px">
-            <span class="alias-activity">{{ alias_info.nb_forward }}</span> forwards,
-            <span class="alias-activity">{{ alias_info.nb_blocked }}</span> blocks,
-            <span class="alias-activity">{{ alias_info.nb_reply }}</span> replies
-            <a href="{{ url_for('dashboard.alias_log', alias_id=gen_email.id) }}"
-               class="btn btn-sm btn-link">
-              See All Activity &nbsp;→
-            </a>
-          </div>
-
-          {% if mailboxes|length > 1 %}
-            <form method="post">
-              <div class="small-text mt-2">Current mailbox</div>
-              <div class="d-flex">
-                <div class="flex-grow-1 mr-2">
-                  <select class="form-control form-control-sm custom-select" name="mailbox">
-                    {% for mailbox in mailboxes %}
-                      <option value="{{ mailbox }}" {% if mailbox == alias_info.mailbox.email %} selected {% endif %}>
-                        {{ mailbox }}
-                      </option>
-                    {% endfor %}
-                  </select>
-                </div>
-
-                <div class="">
-                  <input type="hidden" name="form-name" value="set-mailbox">
-                  <input type="hidden" name="gen-email-id" value="{{ gen_email.id }}">
-
-                  <button class="btn btn-sm btn-outline-info w-100">
-                    Update
-                  </button>
-                </div>
-
-              </div>
-            </form>
-          {% elif alias_info.mailbox != None and alias_info.mailbox.email != current_user.email %}
-            <div class="small-text">
-              Owned by <b>{{ alias_info.mailbox.email }}</b> mailbox
-            </div>
-          {% endif %}
-
-
-          <form method="post">
-            <div class="d-flex mt-2">
-
-              <div class="flex-grow-1 mr-2">
-                <textarea
-                    name="note"
-                    class="form-control"
-                    rows="2"
-                    placeholder="Alias Note.">{{ gen_email.note or "" }}</textarea>
-              </div>
-
-              <div class="">
-                <input type="hidden" name="form-name" value="set-note">
-                <input type="hidden" name="gen-email-id" value="{{ gen_email.id }}">
-
-                <button class="btn btn-sm btn-outline-success w-100">
-                  Save
-                </button>
-              </div>
-            </div>
-          </form>
-
-          <div class="row mt-3">
-            <div class="col">
-              {% if gen_email.enabled %}
-                <a href="{{ url_for('dashboard.alias_contact_manager', alias_id=gen_email.id) }}"
-                    {% if alias_info.show_intro_test_send_email %}
-                   data-intro="Not only alias can receive emails, it can <em>send</em> emails too! <br><br>
-                   You can add a new <em>contact</em> to for your alias here. <br><br>
-                   To send an email to your contact, SimpleLogin will create a <em>special</em> email address. <br><br>
-                   Sending an email to this email address will <em>forward</em> the email to your contact"
-                   data-step="4"
-                    {% endif %}
-                   class="btn btn-sm btn-outline-primary"
-                   data-toggle="tooltip"
-                   title="Not only an alias can receive emails, it can send emails too"
-                >
-                  Send Email&nbsp; &nbsp;<i class="fe fe-send"></i>
-                </a>
-              {% endif %}
-            </div>
-
-            <div class="col">
-              <form method="post">
-                <input type="hidden" name="form-name" value="delete-email">
-                <input type="hidden" name="gen-email-id" value="{{ gen_email.id }}">
-                <input type="hidden" name="alias" class="alias" value="{{ gen_email.email }}">
-
-                <span class="delete-email btn btn-link btn-sm float-right text-danger">
-                  Delete&nbsp; &nbsp;<i class="dropdown-icon fe fe-trash-2 text-danger"></i>
-                </span>
-              </form>
-            </div>
-
-          </div>
-
-        </div>
-      </div>
-    {% endfor %}
-  </div>
-
-
-  {% if  client_users %}
-    <div class="page-header row">
-      <h3 class="page-title col"
-          data-intro="Here you can find the list of website/app on which
-    you have used the <em>Connect with SimpleLogin</em> button <br><br>
-    You also see what information that SimpleLogin has communicated to these website/app when you sign in."
-          data-step="5"
-      >
-        Apps
-      </h3>
-    </div>
-
-    <div class="row row-cards row-deck mt-4">
-      <div class="col-12">
-        <div class="card">
-          <div class="table-responsive">
-            <table class="table table-hover table-outline table-vcenter text-nowrap card-table">
-              <thead>
-              <tr>
-                <th>
-                  App
-                </th>
-                <th>
-                  Info
-                  <i class="fe fe-help-circle" data-toggle="tooltip"
-                     title="Info sent to this app/website"></i>
-                </th>
-                <th class="text-center">
-                  First used
-                  <i class="fe fe-help-circle" data-toggle="tooltip"
-                     title="The first time you have used the SimpleLogin on this app/website"></i>
-                </th>
-                <!--<th class="text-center">Last used</th>-->
-              </tr>
-              </thead>
-              <tbody>
-              {% for client_user in client_users %}
-                <tr>
-                  <td>
-                    {{ client_user.client.name }}
-                  </td>
-
-                  <td>
-                    {% for scope, val in client_user.get_user_info().items() %}
-                      <div>
-                        {% if scope == "email" %}
-                          Email: <a href="mailto:{{ val }}">{{ val }}</a>
-                        {% elif scope == "name" %}
-                          Name: {{ val }}
-                        {% endif %}
-                      </div>
-                    {% endfor %}
-                  </td>
-
-                  <td class="text-center">
-                    {{ client_user.created_at | dt }}
-                  </td>
-
-                </tr>
-              {% endfor %}
-
-              </tbody>
-            </table>
-          </div>
-        </div>
-      </div>
-    </div>
-  {% endif %}
-{% endblock %}
-
-{% block script %}
-  <script>
-    var clipboard = new ClipboardJS('.clipboard');
-
-    var introShown = store.get("introShown");
-    if ("yes" !== introShown) {
-      // only show intro when screen is big enough to show "developer" tab
-      if (window.innerWidth >= 1024) {
-        introJs().start();
-        store.set("introShown", "yes")
-      }
-    }
-
-
-    $(".delete-email").on("click", function (e) {
-      let alias = $(this).parent().find(".alias").val();
-      notie.confirm({
-        text: `Once <b>${alias}</b> is deleted, people/apps ` +
-          "who used to contact you via this alias cannot reach you any more," +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-
-    $(".trigger-email").on("click", function (e) {
-      notie.confirm({
-        text: "SimpleLogin server will send an email to this alias " +
-          "and it will arrive to your inbox, please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-
-    $(".custom-switch-input").change(function (e) {
-      var message = "";
-      let alias = $(this).parent().find(".alias").val();
-
-      if (e.target.checked) {
-        message = `After this, you will start receiving email sent to <b>${alias}</b>, please confirm.`;
-      } else {
-        message = `After this, you will stop receiving email sent to <b>${alias}</b>, please confirm.`;
-      }
-
-      notie.confirm({
-        text: message,
-        cancelCallback: () => {
-          // reset to the original value
-          var oldValue = !$(this).prop("checked");
-          $(this).prop("checked", oldValue);
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    })
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/lifetime_licence.html

@@ -1,29 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "dashboard" %}
-
-{% block title %}
-  Lifetime Licence
-{% endblock %}
-
-{% block default_content %}
-  <div class="bg-white p-6" style="max-width: 60em; margin: auto">
-    <h1 class="h2">Lifetime Licence</h1>
-
-    <div class="mb-4">
-      If you have a lifetime licence, please paste it here. <br>
-      For information, we offer free premium account for education (student, professor or technical staff working at
-      an educational institute). <br>
-      Drop us an email at <a href="mailto:hi@simplelogin.io">hi@simplelogin.io</a> with your student ID or certificate to get the lifetime licence.
-    </div>
-
-    <form method="post">
-      {{ coupon_form.csrf_token }}
-
-      {{ coupon_form.code(class="form-control", placeholder="Licence Code") }}
-      {{ render_field_errors(coupon_form.code) }}
-      <button class="btn btn-success mt-2">Apply</button>
-    </form>
-  </div>
-
-{% endblock %}

app/dashboard/templates/dashboard/mailbox.html

@@ -1,138 +0,0 @@
-{% extends 'default.html' %}
-{% set active_page = "mailbox" %}
-
-{% block title %}
-  Mailboxes
-{% endblock %}
-
-{% block default_content %}
-  <div class="row">
-    <div class="col-md-8 offset-md-2">
-      <h1 class="h3"> Mailboxes </h1>
-
-      {% if not current_user.is_premium() %}
-        <div class="alert alert-danger" role="alert">
-          This feature is only available in premium plan.
-        </div>
-      {% endif %}
-
-      <div class="alert alert-primary" role="alert">
-        A <em>mailbox</em> is just another personal email address. When creating a new alias, you could choose the
-        mailbox that <em>owns</em> this alias, i.e: <br>
-        - all emails sent to this alias will be forwarded to this mailbox <br>
-        - from this mailbox, you can reply/send emails from the alias. <br><br>
-
-        When you signed up, a mailbox is automatically created with your email <b>{{ current_user.email }}</b>
-        <br><br>
-
-        The mailbox doesn't have to be your email: it can be your friend's email
-        if you want to create aliases for your buddy.
-      </div>
-
-      {% for mailbox in mailboxes %}
-        <div class="card" style="max-width: 50rem">
-          <div class="card-body">
-            <h5 class="card-title">
-              {{ mailbox.email }}
-              {% if mailbox.verified %}
-                <span class="cursor" data-toggle="tooltip" data-original-title="Mailbox Verified">✅</span>
-              {% else %}
-                <span class="cursor" data-toggle="tooltip" data-original-title="Mailbox Not Verified">
-                  🚫
-                </span>
-              {% endif %}
-              {% if mailbox.pgp_finger_print %}
-                <span class="cursor" data-toggle="tooltip" data-original-title="PGP Enabled">🗝</span>
-              {% endif %}
-
-              {% if mailbox.id == current_user.default_mailbox_id %}
-                <div class="badge badge-primary float-right" data-toggle="tooltip"
-                     title="When a new random alias is created, it belongs to the default mailbox">Default Mailbox
-                </div>
-              {% endif %}
-            </h5>
-
-            <h6 class="card-subtitle mb-2 text-muted">
-              Created {{ mailbox.created_at | dt }} <br>
-              <span class="font-weight-bold">{{ mailbox.nb_alias() }}</span> aliases. <br>
-
-            </h6>
-
-            <a href="{{ url_for('dashboard.mailbox_detail_route', mailbox_id=mailbox.id) }}">Edit ➡</a>
-          </div>
-
-          <div class="card-footer p-0">
-            <div class="row">
-              {% if mailbox.verified %}
-                <div class="col">
-                  <form method="post">
-                    <input type="hidden" name="form-name" value="set-default">
-                    <input type="hidden" class="mailbox" value="{{ mailbox.email }}">
-                    <input type="hidden" name="mailbox-id" value="{{ mailbox.id }}">
-                    <button class="card-link btn btn-link
-                    {% if mailbox.id == current_user.default_mailbox_id %} disabled {% endif %}"
-                    >
-                      Set As Default Mailbox
-                    </button>
-                  </form>
-                </div>
-              {% endif %}
-
-              <div class="col">
-                <form method="post">
-                  <input type="hidden" name="form-name" value="delete">
-                  <input type="hidden" class="mailbox" value="{{ mailbox.email }}">
-                  <input type="hidden" name="mailbox-id" value="{{ mailbox.id }}">
-                  <span class="card-link btn btn-link text-danger float-right delete-mailbox
-                    {% if mailbox.id == current_user.default_mailbox_id %} disabled {% endif %}">
-                    Delete
-                  </span>
-                </form>
-              </div>
-            </div>
-
-          </div>
-
-        </div>
-      {% endfor %}
-
-      {% if mailboxs|length > 0 %}
-        <hr>
-      {% endif %}
-
-      <form method="post" class="mt-6">
-        {{ new_mailbox_form.csrf_token }}
-        <input type="hidden" name="form-name" value="create">
-
-        <div class="font-weight-bold">Email</div>
-        <div class="small-text">
-          A verification email will be sent to this email to make sure you have access to this email.
-        </div>
-
-        {{ new_mailbox_form.email(class="form-control", placeholder="email@example.com") }}
-        {{ render_field_errors(new_mailbox_form.email) }}
-        <button class="btn btn-lg btn-success mt-2">Create</button>
-      </form>
-
-    </div>
-
-  </div>
-{% endblock %}
-
-{% block script %}
-  <script>
-    $(".delete-mailbox").on("click", function (e) {
-      let mailbox = $(this).parent().find(".mailbox").val();
-      notie.confirm({
-        text: `All aliases owned by this mailbox <b>${mailbox}</b> will be also deleted, ` +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-  </script>
-{% endblock %}

app/dashboard/templates/dashboard/mailbox_detail.html

@@ -1,100 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "mailbox" %}
-
-{% block title %}
-  Mailbox {{ mailbox.email }}
-{% endblock %}
-
-{% block default_content %}
-
-  <div class="col-md-8 offset-md-2 pb-3">
-    <h1 class="h3">{{ mailbox.email }}
-      {% if mailbox.verified %}
-        <span class="cursor" data-toggle="tooltip" data-original-title="Mailbox Verified">✅</span>
-      {% else %}
-        <span class="cursor" data-toggle="tooltip" data-original-title="Mailbox Not Verified">
-          🚫
-        </span>
-      {% endif %}
-      {% if mailbox.pgp_finger_print %}
-        <span class="cursor" data-toggle="tooltip" data-original-title="PGP Enabled">🗝</span>
-      {% endif %}
-    </h1>
-
-    {% if not mailbox.verified %}
-      <div class="alert alert-info">
-        Mailbox not verified, please check your inbox/spam folder for the verification email.
-        <br>
-        To receive the verification email again, you can delete and re-add the mailbox.
-      </div>
-    {% endif %}
-
-    <!-- Change email -->
-    <div class="card">
-      <form method="post" enctype="multipart/form-data">
-        <input type="hidden" name="form-name" value="update-email">
-        {{ change_email_form.csrf_token }}
-
-        <div class="card-body">
-          <div class="card-title">
-            Change Mailbox Address
-          </div>
-          <div class="form-group">
-            <label class="form-label">Address</label>
-
-            <!-- Not allow user to change mailbox if there's a pending change -->
-            {{ change_email_form.email(class="form-control", value=mailbox.email, readonly=pending_email != None) }}
-            {{ render_field_errors(change_email_form.email) }}
-
-            {% if pending_email %}
-              <div class="mt-2">
-                <span class="text-danger">Pending change: {{ pending_email }}</span>
-                <a href="{{ url_for('dashboard.cancel_mailbox_change_route', mailbox_id=mailbox.id) }}"
-                   class="btn btn-secondary btn-sm">
-                  Cancel mailbox change
-                </a>
-              </div>
-            {% endif %}
-          </div>
-          <button class="btn btn-primary">Change</button>
-        </div>
-      </form>
-    </div>
-    <!-- END Change email -->
-
-
-    <!-- Change PGP Public key -->
-    {% if current_user.can_use_pgp %}
-    <div class="card">
-      <form method="post">
-        <input type="hidden" name="form-name" value="pgp">
-
-        <div class="card-body">
-          <div class="card-title">
-            Pretty Good Privacy (PGP)
-            <div class="small-text">
-              By importing your PGP Public Key into SimpleLogin, all emails sent to {{mailbox.email}} are <b>encrypted</b> with your key.
-            </div>
-          </div>
-
-          <div class="form-group">
-            <label class="form-label">PGP Public Key</label>
-
-            <textarea name="pgp" class="form-control" rows=10 placeholder="-----BEGIN PGP PUBLIC KEY BLOCK-----">{{mailbox.pgp_public_key or ""}}</textarea>
-          </div>
-
-          <button class="btn btn-primary" name="action" value="save">Save</button>
-          <button class="btn btn-danger float-right" name="action" value="remove">Remove</button>
-
-        </div>
-      </form>
-
-    </div>
-    {% endif %}
-    <!-- END PGP Public key -->
-
-  </div>
-{% endblock %}
-
-

app/dashboard/templates/dashboard/mfa_cancel.html

@@ -1,28 +0,0 @@
-{% extends 'default.html' %}
-{% set active_page = "setting" %}
-{% block title %}
-  Cancel MFA
-{% endblock %}
-
-
-{% block default_content %}
-  <div class="bg-white p-6" style="max-width: 60em; margin: auto">
-    <h1 class="h2">Multi Factor Authentication</h1>
-    <p>
-      To cancel MFA, please enter the 6-digit number in your TOTP application (Google Authenticator, Authy, etc) here.
-    </p>
-
-    <form method="post">
-      {{ otp_token_form.csrf_token }}
-
-      <div class="font-weight-bold mt-5">Token</div>
-      <div class="small-text">The 6-digit number displayed on your phone.</div>
-
-      {{ otp_token_form.token(class="form-control", autofocus="true") }}
-      {{ render_field_errors(otp_token_form.token) }}
-      <button class="btn btn-lg btn-danger mt-2">Cancel MFA</button>
-    </form>
-
-
-  </div>
-{% endblock %}

app/dashboard/templates/dashboard/mfa_setup.html

@@ -1,51 +0,0 @@
-{% extends 'default.html' %}
-{% set active_page = "setting" %}
-{% block title %}
-  MFA Setup
-{% endblock %}
-
-{% block head %}
-  <script src="{{ url_for('static', filename='node_modules/qrious/dist/qrious.min.js') }}"></script>
-{% endblock %}
-
-{% block default_content %}
-  <div class="bg-white p-6" style="max-width: 60em; margin: auto">
-    <h1 class="h2">Multi Factor Authentication</h1>
-    <p>Please open a TOTP application (Google Authenticator, Authy, etc)
-      on your smartphone and scan the following QR Code:
-    </p>
-
-    <canvas id="qr"></canvas>
-
-    <script>
-      (function () {
-        var qr = new QRious({
-          element: document.getElementById('qr'),
-          value: '{{otp_uri}}'
-        });
-      })();
-    </script>
-
-    <div class="mt-3 mb-2">
-      Or you can use the manual entry with the following key:
-    </div>
-
-    <div class="mb-3 p-3" style="background-color: #eee">
-      {{ current_user.otp_secret }}
-    </div>
-
-
-    <form method="post">
-      {{ otp_token_form.csrf_token }}
-
-      <div class="font-weight-bold mt-5">Token</div>
-      <div class="small-text">Please enter the 6-digit number displayed on your phone.</div>
-
-      {{ otp_token_form.token(class="form-control", placeholder="") }}
-      {{ render_field_errors(otp_token_form.token) }}
-      <button class="btn btn-lg btn-success mt-2">Validate</button>
-    </form>
-
-
-  </div>
-{% endblock %}

app/dashboard/templates/dashboard/pricing.html

@@ -1,101 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "dashboard" %}
-
-{% block title %}
-  Pricing
-{% endblock %}
-
-{% block head %}
-  <script src="https://cdn.paddle.com/paddle/paddle.js"></script>
-  <script>
-    if (window.Paddle === undefined) {
-      console.log("cannot load Paddle from CDN");
-      document.write('<script src="/static/vendor/paddle.js"><\/script>')
-    }
-  </script>
-
-{% endblock %}
-
-{% block default_content %}
-  <div class="row">
-    <div class="col-sm-6 col-lg-6">
-      <div class="card">
-        <div class="card-body text-center">
-          <div class="h3">Premium</div>
-
-          <ul class="list-unstyled leading-loose mb-3">
-            <li><i class="fe fe-check text-success mr-2" aria-hidden="true"></i> Unlimited Alias</li>
-            <li><i class="fe fe-check text-success mr-2" aria-hidden="true"></i>
-              Custom Domain
-            </li>
-            <li><i class="fe fe-check text-success mr-2" aria-hidden="true"></i>
-              Catch-all (or wildcard) alias
-            </li>
-            <li><i class="fe fe-check text-success mr-2" aria-hidden="true"></i>
-              Directory (or Username)
-            </li>
-            <li><i class="fe fe-check text-success mr-2" aria-hidden="true"></i>
-              Multiple Mailboxes
-            </li>
-          </ul>
-
-          <div class="small-text">More info on our <a href="https://simplelogin.io/pricing" target="_blank">Pricing
-            Page <i class="fe fe-external-link"></i>
-          </a></div>
-        </div>
-      </div>
-    </div>
-
-    <div class="col-sm-6 col-lg-6">
-      <div class="display-6 my-3">
-        🔐 Secure payments by
-        <a href="https://paddle.com" target="_blank">Paddle<i class="fe fe-external-link"></i></a></li>
-        </a>
-      </div>
-
-      {% if current_user.is_cancel() %}
-      <div class="alert alert-primary" role="alert">
-        You have an active subscription until {{current_user.next_bill_date()}}. <br>
-        Please note that if you re-subscribe now, this will be a completely
-        new subscription and
-        your payment method will be charged <b>immediately</b>.
-      </div>
-      {% endif %}
-
-      <div class="mb-3">
-        Paddle supported payment methods include bank cards (Mastercard, Visa, American Express, etc) or PayPal. <br>
-        Send us an email at <a href="mailto:hi@simplelogin.io">hi@simplelogin.io</a> if you need other payment options
-        (e.g. IBAN transfer).
-      </div>
-
-      <button class="btn btn-success" onclick="upgrade({{ PADDLE_MONTHLY_PRODUCT_ID }})">
-        Monthly <br>
-        $2.99/month
-      </button>
-
-      <button class="btn btn-primary" onclick="upgrade({{ PADDLE_YEARLY_PRODUCT_ID }})">
-        Yearly <br>
-        $29.99/year
-      </button>
-
-      <hr class="my-6">
-      If you have a lifetime licence, please go to this page to apply your licence code.
-      <a href="{{ url_for('dashboard.lifetime_licence') }}">Lifetime Licence</a>
-    </div>
-  </div>
-
-
-  <script type="text/javascript">
-    Paddle.Setup({vendor: {{ PADDLE_VENDOR_ID }}});
-
-    function upgrade(productId) {
-      Paddle.Checkout.open({
-        product: productId,
-        email: "{{ current_user.email }}",
-        success: "{{ success_url }}"
-      });
-    }
-  </script>
-
-{% endblock %}

app/dashboard/templates/dashboard/setting.html

@@ -1,236 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "setting" %}
-
-{% block title %}
-  Settings
-{% endblock %}
-
-{% block default_content %}
-
-  <div class="col-md-8 offset-md-2 pb-3">
-    <!-- Change email -->
-    <div class="card">
-      <form method="post" enctype="multipart/form-data">
-        <input type="hidden" name="form-name" value="update-email">
-        {{ change_email_form.csrf_token }}
-
-        <div class="card-body">
-          <div class="card-title">
-            Change Email Address
-          </div>
-          <div class="form-group">
-            <label class="form-label">Email</label>
-
-            <!-- Not allow user to change email if there's a pending change -->
-            {{ change_email_form.email(class="form-control", value=current_user.email, readonly=pending_email != None) }}
-            {{ render_field_errors(change_email_form.email) }}
-
-            {% if pending_email %}
-              <div class="mt-2">
-                <span class="text-danger">Pending email change: {{ pending_email }}</span>
-                <a href="{{ url_for('dashboard.resend_email_change') }}" class="btn btn-secondary btn-sm">Resend
-                  confirmation email</a>
-                <a href="{{ url_for('dashboard.cancel_email_change') }}" class="btn btn-secondary btn-sm">Cancel email
-                  change</a>
-              </div>
-            {% endif %}
-          </div>
-          <button class="btn btn-primary">Change Email</button>
-        </div>
-      </form>
-    </div>
-    <!-- END Change email -->
-
-    <!-- Change name & profile picture -->
-    <div class="card">
-      <form method="post" enctype="multipart/form-data">
-        {{ form.csrf_token }}
-        <input type="hidden" name="form-name" value="update-profile">
-
-        <div class="card-body">
-          <div class="card-title">
-            Change Profile
-          </div>
-          <div class="form-group">
-            <label class="form-label">Name</label>
-            {{ form.name(class="form-control", value=current_user.name) }}
-            {{ render_field_errors(form.name) }}
-          </div>
-
-          <div class="form-group">
-            <div class="form-label">Profile picture</div>
-            {{ form.profile_picture(class="form-control-file") }}
-            {{ render_field_errors(form.profile_picture) }}
-            {% if current_user.profile_picture_id %}
-              <img src="{{ current_user.profile_picture_url() }}" class="profile-picture">
-            {% endif %}
-          </div>
-          <button class="btn btn-primary">Update</button>
-        </div>
-      </form>
-    </div>
-    <!-- END change name & profile picture -->
-
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title">Multi-Factor Authentication (MFA)
-          <div class="small-text mt-1 mb-3">
-            Secure your account with Multi-Factor Authentication. <br>
-            This requires having applications like Google Authenticator, Authy, FreeOTP, etc.
-          </div>
-        </div>
-        {% if not current_user.enable_otp %}
-          <a href="{{ url_for('dashboard.mfa_setup') }}" class="btn btn-outline-primary">Enable</a>
-        {% else %}
-          <a href="{{ url_for('dashboard.mfa_cancel') }}" class="btn btn-outline-danger">Cancel MFA</a>
-        {% endif %}
-      </div>
-    </div>
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title">
-          Change password
-          <div class="small-text mt-1 mb-3">
-            You will receive an email containing instructions on how to change password.
-          </div>
-        </div>
-        <form method="post">
-          <input type="hidden" name="form-name" value="change-password">
-          <button class="btn btn-outline-primary">Change password</button>
-        </form>
-      </div>
-    </div>
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title">Random Alias
-          <div class="small-text mt-1 mb-3">Choose how to create your email alias by default</div>
-        </div>
-        <form method="post" class="form-inline">
-          <input type="hidden" name="form-name" value="change-alias-generator">
-          <select class="custom-select mr-sm-2" name="alias-generator-scheme">
-            <option value="{{ AliasGeneratorEnum.word.value }}"
-                {% if current_user.alias_generator == AliasGeneratorEnum.word.value %} selected {% endif %} >Based on
-              Random {{ AliasGeneratorEnum.word.name.capitalize() }}</option>
-            <option value="{{ AliasGeneratorEnum.uuid.value }}"
-                {% if current_user.alias_generator == AliasGeneratorEnum.uuid.value %} selected {% endif %} >Based
-              on {{ AliasGeneratorEnum.uuid.name.upper() }}</option>
-          </select>
-          <button class="btn btn-outline-primary">Update Preference</button>
-        </form>
-      </div>
-    </div>
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title" id="notification">Newsletters
-          <div class="small-text mt-1 mb-3">
-            Every now and then we can send you an email
-            to let you know about a new feature that might be useful to you.
-          </div>
-        </div>
-        <form method="post">
-          <input type="hidden" name="form-name" value="notification-preference">
-          <div class="form-check">
-            <input type="checkbox" id="notification" name="notification" {% if current_user.notification %}
-                   checked {% endif %} class="form-check-input">
-            <label for="notification">I want to receive your newsletter</label>
-          </div>
-          <button type="submit" class="btn btn-outline-primary">Submit</button>
-        </form>
-      </div>
-    </div>
-
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title mb-3">Current Plan</div>
-
-        {% if current_user.get_subscription() %}
-          You are on the  {{ current_user.get_subscription().plan_name() }} plan. <br>
-          <a href="{{ url_for('dashboard.billing') }}" class="btn btn-outline-primary">
-            Manage Subscription
-          </a>
-        {% elif manual_sub %}
-          You are on the Premium plan. The plan ends {{ manual_sub.end_at | dt }}.
-        {% elif current_user.lifetime %}
-          You have the lifetime licence.
-        {% elif current_user.in_trial() %}
-          You are in the trial period. The trial ends {{ current_user.trial_end | dt }}.
-        {% else %}
-          You are on the Free plan.
-        {% endif %}
-      </div>
-    </div>
-
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title">Deleted Aliases
-          <div class="small-text mt-1 mb-3" style="max-width: 40rem">
-            When an alias is deleted, all its activities are deleted and no emails can be sent to it. <br>
-            It is moved to another location and only used to check when new alias is created. <br>
-            This check is necessary to avoid someone else accidentally taking this alias. <br>
-            Because in this case, the other person might receive inadvertently information that belong to you. <br>
-          </div>
-        </div>
-        <a href="{{ url_for('dashboard.deleted_alias_route') }}" class="btn btn-outline-primary">
-          See deleted aliases
-        </a>
-      </div>
-    </div>
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title">Export Data
-          <div class="small-text mt-1 mb-3">
-            You can download all aliases you have created on SimpleLogin along with other data.
-          </div>
-        </div>
-        <form method="post">
-          <input type="hidden" name="form-name" value="export-data">
-          <button class="btn btn-outline-info">Export Data</button>
-        </form>
-      </div>
-    </div>
-
-
-    <div class="card">
-      <div class="card-body">
-        <div class="card-title">Delete Account
-          <div class="small-text mt-1 mb-3">Please note that this operation is irreversible.
-          </div>
-
-        </div>
-        <form method="post">
-          <input type="hidden" name="form-name" value="delete-account">
-          <span class="delete-account btn btn-outline-danger">Delete account</span>
-        </form>
-      </div>
-    </div>
-
-  </div>
-
-{% endblock %}
-
-{% block script %}
-  <script>
-    $(".delete-account").on("click", function (e) {
-      notie.confirm({
-        text: "All your data including your aliases will be deleted, " +
-          "other people might not be able to reach you after, " +
-          " please confirm.",
-        cancelCallback: () => {
-          // nothing to do
-        },
-        submitCallback: () => {
-          $(this).closest("form").submit();
-        }
-      });
-    });
-  </script>
-{% endblock %}
-

app/dashboard/templates/dashboard/unsubscribe.html

@@ -1,28 +0,0 @@
-{% extends 'default.html' %}
-
-{% set active_page = "dashboard" %}
-
-{% block title %}
-  Block an alias
-{% endblock %}
-
-{% block default_content %}
-
-  <div class="col-md-6 offset-md-3 text-center bg-white p-3 mt-5">
-    <h1 class="h3">
-      Block alias
-    </h1>
-    <p>
-      You are about to block the alias <a href="mailto:{{alias}}">{{alias}}</a>
-    </p>
-    <p>
-      After this, you will stop receiving all emails sent to this alias, please confirm.
-    </p>
-
-    <form method="post">
-      <button class="btn btn-warning">Confirm</button>
-    </form>
-  </div>
-
-{% endblock %}
-

app/dashboard/views/account_setting.py

@@ -0,0 +1,242 @@
+import arrow
+from flask import (
+    render_template,
+    request,
+    redirect,
+    url_for,
+    flash,
+)
+from flask_login import login_required, current_user
+
+from app import email_utils
+from app.config import (
+    URL,
+    FIRST_ALIAS_DOMAIN,
+    ALIAS_RANDOM_SUFFIX_LENGTH,
+    CONNECT_WITH_PROTON,
+)
+from app.dashboard.base import dashboard_bp
+from app.dashboard.views.enter_sudo import sudo_required
+from app.dashboard.views.mailbox_detail import ChangeEmailForm
+from app.db import Session
+from app.email_utils import (
+    email_can_be_used_as_mailbox,
+    personal_email_already_used,
+)
+from app.extensions import limiter
+from app.jobs.export_user_data_job import ExportUserDataJob
+from app.log import LOG
+from app.models import (
+    BlockBehaviourEnum,
+    PlanEnum,
+    ResetPasswordCode,
+    EmailChange,
+    User,
+    Alias,
+    AliasGeneratorEnum,
+    SenderFormatEnum,
+    UnsubscribeBehaviourEnum,
+)
+from app.proton.utils import perform_proton_account_unlink
+from app.utils import (
+    random_string,
+    CSRFValidationForm,
+    canonicalize_email,
+)
+
+
+@dashboard_bp.route("/account_setting", methods=["GET", "POST"])
+@login_required
+@sudo_required
+@limiter.limit("5/minute", methods=["POST"])
+def account_setting():
+    change_email_form = ChangeEmailForm()
+    csrf_form = CSRFValidationForm()
+
+    email_change = EmailChange.get_by(user_id=current_user.id)
+    if email_change:
+        pending_email = email_change.new_email
+    else:
+        pending_email = None
+
+    if request.method == "POST":
+        if not csrf_form.validate():
+            flash("Invalid request", "warning")
+            return redirect(url_for("dashboard.setting"))
+        if request.form.get("form-name") == "update-email":
+            if change_email_form.validate():
+                # whether user can proceed with the email update
+                new_email_valid = True
+                new_email = canonicalize_email(change_email_form.email.data)
+                if new_email != current_user.email and not pending_email:
+                    # check if this email is not already used
+                    if personal_email_already_used(new_email) or Alias.get_by(
+                        email=new_email
+                    ):
+                        flash(f"Email {new_email} already used", "error")
+                        new_email_valid = False
+                    elif not email_can_be_used_as_mailbox(new_email):
+                        flash(
+                            "You cannot use this email address as your personal inbox.",
+                            "error",
+                        )
+                        new_email_valid = False
+                    # a pending email change with the same email exists from another user
+                    elif EmailChange.get_by(new_email=new_email):
+                        other_email_change: EmailChange = EmailChange.get_by(
+                            new_email=new_email
+                        )
+                        LOG.w(
+                            "Another user has a pending %s with the same email address. Current user:%s",
+                            other_email_change,
+                            current_user,
+                        )
+
+                        if other_email_change.is_expired():
+                            LOG.d(
+                                "delete the expired email change %s", other_email_change
+                            )
+                            EmailChange.delete(other_email_change.id)
+                            Session.commit()
+                        else:
+                            flash(
+                                "You cannot use this email address as your personal inbox.",
+                                "error",
+                            )
+                            new_email_valid = False
+
+                    if new_email_valid:
+                        email_change = EmailChange.create(
+                            user_id=current_user.id,
+                            code=random_string(
+                                60
+                            ),  # todo: make sure the code is unique
+                            new_email=new_email,
+                        )
+                        Session.commit()
+                        send_change_email_confirmation(current_user, email_change)
+                        flash(
+                            "A confirmation email is on the way, please check your inbox",
+                            "success",
+                        )
+                        return redirect(url_for("dashboard.account_setting"))
+        elif request.form.get("form-name") == "change-password":
+            flash(
+                "You are going to receive an email containing instructions to change your password",
+                "success",
+            )
+            send_reset_password_email(current_user)
+            return redirect(url_for("dashboard.account_setting"))
+        elif request.form.get("form-name") == "send-full-user-report":
+            if ExportUserDataJob(current_user).store_job_in_db():
+                flash(
+                    "You will receive your SimpleLogin data via email shortly",
+                    "success",
+                )
+            else:
+                flash("An export of your data is currently in progress", "error")
+
+    partner_sub = None
+    partner_name = None
+
+    return render_template(
+        "dashboard/account_setting.html",
+        csrf_form=csrf_form,
+        PlanEnum=PlanEnum,
+        SenderFormatEnum=SenderFormatEnum,
+        BlockBehaviourEnum=BlockBehaviourEnum,
+        change_email_form=change_email_form,
+        pending_email=pending_email,
+        AliasGeneratorEnum=AliasGeneratorEnum,
+        UnsubscribeBehaviourEnum=UnsubscribeBehaviourEnum,
+        partner_sub=partner_sub,
+        partner_name=partner_name,
+        FIRST_ALIAS_DOMAIN=FIRST_ALIAS_DOMAIN,
+        ALIAS_RAND_SUFFIX_LENGTH=ALIAS_RANDOM_SUFFIX_LENGTH,
+        connect_with_proton=CONNECT_WITH_PROTON,
+    )
+
+
+def send_reset_password_email(user):
+    """
+    generate a new ResetPasswordCode and send it over email to user
+    """
+    # the activation code is valid for 1h
+    reset_password_code = ResetPasswordCode.create(
+        user_id=user.id, code=random_string(60)
+    )
+    Session.commit()
+
+    reset_password_link = f"{URL}/auth/reset_password?code={reset_password_code.code}"
+
+    email_utils.send_reset_password_email(user.email, reset_password_link)
+
+
+def send_change_email_confirmation(user: User, email_change: EmailChange):
+    """
+    send confirmation email to the new email address
+    """
+
+    link = f"{URL}/auth/change_email?code={email_change.code}"
+
+    email_utils.send_change_email(email_change.new_email, user.email, link)
+
+
+@dashboard_bp.route("/resend_email_change", methods=["GET", "POST"])
+@limiter.limit("5/hour")
+@login_required
+@sudo_required
+def resend_email_change():
+    form = CSRFValidationForm()
+    if not form.validate():
+        flash("Invalid request. Please try again", "warning")
+        return redirect(url_for("dashboard.setting"))
+    email_change = EmailChange.get_by(user_id=current_user.id)
+    if email_change:
+        # extend email change expiration
+        email_change.expired = arrow.now().shift(hours=12)
+        Session.commit()
+
+        send_change_email_confirmation(current_user, email_change)
+        flash("A confirmation email is on the way, please check your inbox", "success")
+        return redirect(url_for("dashboard.setting"))
+    else:
+        flash(
+            "You have no pending email change. Redirect back to Setting page", "warning"
+        )
+        return redirect(url_for("dashboard.setting"))
+
+
+@dashboard_bp.route("/cancel_email_change", methods=["GET", "POST"])
+@login_required
+@sudo_required
+def cancel_email_change():
+    form = CSRFValidationForm()
+    if not form.validate():
+        flash("Invalid request. Please try again", "warning")
+        return redirect(url_for("dashboard.setting"))
+    email_change = EmailChange.get_by(user_id=current_user.id)
+    if email_change:
+        EmailChange.delete(email_change.id)
+        Session.commit()
+        flash("Your email change is cancelled", "success")
+        return redirect(url_for("dashboard.setting"))
+    else:
+        flash(
+            "You have no pending email change. Redirect back to Setting page", "warning"
+        )
+        return redirect(url_for("dashboard.setting"))
+
+
+@dashboard_bp.route("/unlink_proton_account", methods=["POST"])
+@login_required
+@sudo_required
+def unlink_proton_account():
+    csrf_form = CSRFValidationForm()
+    if not csrf_form.validate():
+        flash("Invalid request", "warning")
+        return redirect(url_for("dashboard.setting"))
+
+    perform_proton_account_unlink(current_user)
+    flash("Your Proton account has been unlinked", "success")
+    return redirect(url_for("dashboard.setting"))

app/dashboard/views/alias_contact_manager.py

@@ -1,17 +1,31 @@
-import re
+from dataclasses import dataclass
+from operator import or_
 
-from flask import render_template, request, redirect, url_for, flash
+from flask import render_template, request, redirect, flash
+from flask import url_for
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
+from sqlalchemy import and_, func, case
 from wtforms import StringField, validators, ValidationError
 
-from app.config import EMAIL_DOMAIN
+# Need to import directly from config to allow modification from the tests
+from app import config, parallel_limiter
 from app.dashboard.base import dashboard_bp
-from app.email_utils import get_email_part
-from app.extensions import db
+from app.db import Session
+from app.email_utils import (
+    generate_reply_email,
+    parse_full_address,
+)
+from app.email_validation import is_valid_email
+from app.errors import (
+    CannotCreateContactForReverseAlias,
+    ErrContactErrorUpgradeNeeded,
+    ErrAddressInvalid,
+    ErrContactAlreadyExists,
+)
 from app.log import LOG
-from app.models import GenEmail, ForwardEmail
-from app.utils import random_string
+from app.models import Alias, Contact, EmailLog, User
+from app.utils import sanitize_email, CSRFValidationForm
 
 
 def email_validator():
@@ -31,118 +45,280 @@ def email_validator():
             if email.find("<") + 1 < email.find(">"):
                 email_part = email[email.find("<") + 1 : email.find(">")].strip()
 
-        if re.match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$", email_part):
-            return
-
-        raise ValidationError(message)
+        if not is_valid_email(email_part):
+            raise ValidationError(message)
 
     return _check
 
 
+def create_contact(user: User, alias: Alias, contact_address: str) -> Contact:
+    """
+    Create a contact for a user. Can be restricted for new free users by enabling DISABLE_CREATE_CONTACTS_FOR_FREE_USERS.
+    Can throw exceptions:
+     - ErrAddressInvalid
+     - ErrContactAlreadyExists
+     - ErrContactUpgradeNeeded - If DISABLE_CREATE_CONTACTS_FOR_FREE_USERS this exception will be raised for new free users
+    """
+    if not contact_address:
+        raise ErrAddressInvalid("Empty address")
+    try:
+        contact_name, contact_email = parse_full_address(contact_address)
+    except ValueError:
+        raise ErrAddressInvalid(contact_address)
+
+    contact_email = sanitize_email(contact_email)
+    if not is_valid_email(contact_email):
+        raise ErrAddressInvalid(contact_email)
+
+    contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
+    if contact:
+        raise ErrContactAlreadyExists(contact)
+
+    if not user.can_create_contacts():
+        raise ErrContactErrorUpgradeNeeded()
+
+    contact = Contact.create(
+        user_id=alias.user_id,
+        alias_id=alias.id,
+        website_email=contact_email,
+        name=contact_name,
+        reply_email=generate_reply_email(contact_email, alias),
+    )
+
+    LOG.d(
+        "create reverse-alias for %s %s, reverse alias:%s",
+        contact_address,
+        alias,
+        contact.reply_email,
+    )
+    Session.commit()
+
+    return contact
+
+
 class NewContactForm(FlaskForm):
     email = StringField(
         "Email", validators=[validators.DataRequired(), email_validator()]
     )
 
 
-@dashboard_bp.route("/alias_contact_manager/<alias_id>/", methods=["GET", "POST"])
-@dashboard_bp.route(
-    "/alias_contact_manager/<alias_id>/<int:forward_email_id>", methods=["GET", "POST"]
-)
+@dataclass
+class ContactInfo(object):
+    contact: Contact
+
+    nb_forward: int
+    nb_reply: int
+
+    latest_email_log: EmailLog
+
+
+def get_contact_infos(
+    alias: Alias, page=0, contact_id=None, query: str = ""
+) -> [ContactInfo]:
+    """if contact_id is set, only return the contact info for this contact"""
+    sub = (
+        Session.query(
+            Contact.id,
+            func.sum(case([(EmailLog.is_reply, 1)], else_=0)).label("nb_reply"),
+            func.sum(
+                case(
+                    [
+                        (
+                            and_(
+                                EmailLog.is_reply.is_(False),
+                                EmailLog.blocked.is_(False),
+                            ),
+                            1,
+                        )
+                    ],
+                    else_=0,
+                )
+            ).label("nb_forward"),
+            func.max(EmailLog.created_at).label("max_email_log_created_at"),
+        )
+        .join(
+            EmailLog,
+            EmailLog.contact_id == Contact.id,
+            isouter=True,
+        )
+        .filter(Contact.alias_id == alias.id)
+        .group_by(Contact.id)
+        .subquery()
+    )
+
+    q = (
+        Session.query(
+            Contact,
+            EmailLog,
+            sub.c.nb_reply,
+            sub.c.nb_forward,
+        )
+        .join(
+            EmailLog,
+            EmailLog.contact_id == Contact.id,
+            isouter=True,
+        )
+        .filter(Contact.alias_id == alias.id)
+        .filter(Contact.id == sub.c.id)
+        .filter(
+            or_(
+                EmailLog.created_at == sub.c.max_email_log_created_at,
+                # no email log yet for this contact
+                sub.c.max_email_log_created_at.is_(None),
+            )
+        )
+    )
+
+    if query:
+        q = q.filter(
+            or_(
+                Contact.website_email.ilike(f"%{query}%"),
+                Contact.name.ilike(f"%{query}%"),
+            )
+        )
+
+    if contact_id:
+        q = q.filter(Contact.id == contact_id)
+
+    latest_activity = case(
+        [
+            (EmailLog.created_at > Contact.created_at, EmailLog.created_at),
+            (EmailLog.created_at < Contact.created_at, Contact.created_at),
+        ],
+        else_=Contact.created_at,
+    )
+    q = (
+        q.order_by(latest_activity.desc())
+        .limit(config.PAGE_LIMIT)
+        .offset(page * config.PAGE_LIMIT)
+    )
+
+    ret = []
+    for contact, latest_email_log, nb_reply, nb_forward in q:
+        contact_info = ContactInfo(
+            contact=contact,
+            nb_forward=nb_forward,
+            nb_reply=nb_reply,
+            latest_email_log=latest_email_log,
+        )
+        ret.append(contact_info)
+
+    return ret
+
+
+def delete_contact(alias: Alias, contact_id: int):
+    contact = Contact.get(contact_id)
+
+    if not contact:
+        flash("Unknown error. Refresh the page", "warning")
+    elif contact.alias_id != alias.id:
+        flash("You cannot delete reverse-alias", "warning")
+    else:
+        delete_contact_email = contact.website_email
+        Contact.delete(contact_id)
+        Session.commit()
+
+        flash(f"Reverse-alias for {delete_contact_email} has been deleted", "success")
+
+
+@dashboard_bp.route("/alias_contact_manager/<int:alias_id>/", methods=["GET", "POST"])
 @login_required
-def alias_contact_manager(alias_id, forward_email_id=None):
-    gen_email = GenEmail.get(alias_id)
+@parallel_limiter.lock(name="contact_creation")
+def alias_contact_manager(alias_id):
+    highlight_contact_id = None
+    if request.args.get("highlight_contact_id"):
+        try:
+            highlight_contact_id = int(request.args.get("highlight_contact_id"))
+        except ValueError:
+            flash("Invalid contact id", "error")
+            return redirect(url_for("dashboard.index"))
+
+    alias = Alias.get(alias_id)
+
+    page = 0
+    if request.args.get("page"):
+        page = int(request.args.get("page"))
+
+    query = request.args.get("query") or ""
 
     # sanity check
-    if not gen_email:
+    if not alias:
         flash("You do not have access to this page", "warning")
         return redirect(url_for("dashboard.index"))
 
-    if gen_email.user_id != current_user.id:
+    if alias.user_id != current_user.id:
         flash("You do not have access to this page", "warning")
         return redirect(url_for("dashboard.index"))
 
     new_contact_form = NewContactForm()
+    csrf_form = CSRFValidationForm()
 
     if request.method == "POST":
+        if not csrf_form.validate():
+            flash("Invalid request", "warning")
+            return redirect(request.url)
         if request.form.get("form-name") == "create":
             if new_contact_form.validate():
-                contact_email = new_contact_form.email.data.strip()
-
-                # generate a reply_email, make sure it is unique
-                # not use while to avoid infinite loop
-                for _ in range(1000):
-                    reply_email = f"ra+{random_string(25)}@{EMAIL_DOMAIN}"
-                    if not ForwardEmail.get_by(reply_email=reply_email):
-                        break
-
-                website_email = get_email_part(contact_email)
-
-                # already been added
-                if ForwardEmail.get_by(
-                    gen_email_id=gen_email.id, website_email=website_email
-                ):
-                    flash(f"{website_email} is already added", "error")
-                    return redirect(
-                        url_for("dashboard.alias_contact_manager", alias_id=alias_id)
-                    )
-
-                forward_email = ForwardEmail.create(
-                    gen_email_id=gen_email.id,
-                    website_email=website_email,
-                    website_from=contact_email,
-                    reply_email=reply_email,
-                )
-
-                LOG.d("create reverse-alias for %s", contact_email)
-                db.session.commit()
-                flash(f"Reverse alias for {contact_email} is created", "success")
-
+                contact_address = new_contact_form.email.data.strip()
+                try:
+                    contact = create_contact(current_user, alias, contact_address)
+                except (
+                    ErrContactErrorUpgradeNeeded,
+                    ErrAddressInvalid,
+                    ErrContactAlreadyExists,
+                    CannotCreateContactForReverseAlias,
+                ) as excp:
+                    flash(excp.error_for_user(), "error")
+                    return redirect(request.url)
+                flash(f"Reverse alias for {contact_address} is created", "success")
                 return redirect(
                     url_for(
                         "dashboard.alias_contact_manager",
                         alias_id=alias_id,
-                        forward_email_id=forward_email.id,
+                        highlight_contact_id=contact.id,
                     )
                 )
         elif request.form.get("form-name") == "delete":
-            forward_email_id = request.form.get("forward-email-id")
-            forward_email = ForwardEmail.get(forward_email_id)
-
-            if not forward_email:
-                flash("Unknown error. Refresh the page", "warning")
-                return redirect(
-                    url_for("dashboard.alias_contact_manager", alias_id=alias_id)
-                )
-            elif forward_email.gen_email_id != gen_email.id:
-                flash("You cannot delete reverse-alias", "warning")
-                return redirect(
-                    url_for("dashboard.alias_contact_manager", alias_id=alias_id)
-                )
-
-            contact_name = forward_email.website_from
-            ForwardEmail.delete(forward_email_id)
-            db.session.commit()
-
-            flash(f"Reverse-alias for {contact_name} has been deleted", "success")
-
+            contact_id = request.form.get("contact-id")
+            delete_contact(alias, contact_id)
             return redirect(
                 url_for("dashboard.alias_contact_manager", alias_id=alias_id)
             )
 
-    # make sure highlighted forward_email is at array start
-    forward_emails = gen_email.forward_emails
+        elif request.form.get("form-name") == "search":
+            query = request.form.get("query")
+            return redirect(
+                url_for(
+                    "dashboard.alias_contact_manager",
+                    alias_id=alias_id,
+                    query=query,
+                    highlight_contact_id=highlight_contact_id,
+                )
+            )
 
-    if forward_email_id:
-        forward_emails = sorted(
-            forward_emails, key=lambda fe: fe.id == forward_email_id, reverse=True
+    contact_infos = get_contact_infos(alias, page, query=query)
+    last_page = len(contact_infos) < config.PAGE_LIMIT
+    nb_contact = Contact.filter(Contact.alias_id == alias.id).count()
+
+    # if highlighted contact isn't included, fetch it
+    # make sure highlighted contact is at array start
+    contact_ids = [contact_info.contact.id for contact_info in contact_infos]
+    if highlight_contact_id and highlight_contact_id not in contact_ids:
+        contact_infos = (
+            get_contact_infos(alias, contact_id=highlight_contact_id, query=query)
+            + contact_infos
         )
 
     return render_template(
         "dashboard/alias_contact_manager.html",
-        forward_emails=forward_emails,
-        alias=gen_email.email,
-        gen_email=gen_email,
+        contact_infos=contact_infos,
+        alias=alias,
         new_contact_form=new_contact_form,
-        forward_email_id=forward_email_id,
+        highlight_contact_id=highlight_contact_id,
+        page=page,
+        last_page=last_page,
+        query=query,
+        nb_contact=nb_contact,
+        can_create_contacts=current_user.can_create_contacts(),
+        csrf_form=csrf_form,
     )

app/dashboard/views/alias_export.py

@@ -0,0 +1,13 @@
+from app.dashboard.base import dashboard_bp
+from flask_login import login_required, current_user
+from app.alias_utils import alias_export_csv
+from app.dashboard.views.enter_sudo import sudo_required
+from app.extensions import limiter
+
+
+@dashboard_bp.route("/alias_export", methods=["GET"])
+@login_required
+@sudo_required
+@limiter.limit("2/minute")
+def alias_export_route():
+    return alias_export_csv(current_user)

app/dashboard/views/alias_log.py

@@ -4,19 +4,20 @@ from flask_login import login_required, current_user
 
 from app.config import PAGE_LIMIT
 from app.dashboard.base import dashboard_bp
-from app.extensions import db
-from app.models import GenEmail, ForwardEmailLog, ForwardEmail
+from app.db import Session
+from app.models import Alias, EmailLog, Contact
 
 
 class AliasLog:
     website_email: str
-    website_from: str
+    reverse_alias: str
     alias: str
     when: arrow.Arrow
     is_reply: bool
     blocked: bool
     bounced: bool
-    mailbox: str
+    email_log: EmailLog
+    contact: Contact
 
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
@@ -29,31 +30,31 @@ class AliasLog:
 @dashboard_bp.route("/alias_log/<int:alias_id>/<int:page_id>")
 @login_required
 def alias_log(alias_id, page_id):
-    gen_email = GenEmail.get(alias_id)
+    alias = Alias.get(alias_id)
 
     # sanity check
-    if not gen_email:
+    if not alias:
         flash("You do not have access to this page", "warning")
         return redirect(url_for("dashboard.index"))
 
-    if gen_email.user_id != current_user.id:
+    if alias.user_id != current_user.id:
         flash("You do not have access to this page", "warning")
         return redirect(url_for("dashboard.index"))
 
-    logs = get_alias_log(gen_email, page_id)
+    logs = get_alias_log(alias, page_id)
     base = (
-        db.session.query(ForwardEmail, ForwardEmailLog)
-        .filter(ForwardEmail.id == ForwardEmailLog.forward_id)
-        .filter(ForwardEmail.gen_email_id == gen_email.id)
+        Session.query(Contact, EmailLog)
+        .filter(Contact.id == EmailLog.contact_id)
+        .filter(Contact.alias_id == alias.id)
     )
     total = base.count()
     email_forwarded = (
-        base.filter(ForwardEmailLog.is_reply == False)
-        .filter(ForwardEmailLog.blocked == False)
+        base.filter(EmailLog.is_reply.is_(False))
+        .filter(EmailLog.blocked.is_(False))
         .count()
     )
-    email_replied = base.filter(ForwardEmailLog.is_reply == True).count()
-    email_blocked = base.filter(ForwardEmailLog.blocked == True).count()
+    email_replied = base.filter(EmailLog.is_reply.is_(True)).count()
+    email_blocked = base.filter(EmailLog.blocked.is_(True)).count()
     last_page = (
         len(logs) < PAGE_LIMIT
     )  # lightweight pagination without counting all objects
@@ -61,31 +62,31 @@ def alias_log(alias_id, page_id):
     return render_template("dashboard/alias_log.html", **locals())
 
 
-def get_alias_log(gen_email: GenEmail, page_id=0):
+def get_alias_log(alias: Alias, page_id=0) -> [AliasLog]:
     logs: [AliasLog] = []
-    mailbox = gen_email.mailbox_email()
 
     q = (
-        db.session.query(ForwardEmail, ForwardEmailLog)
-        .filter(ForwardEmail.id == ForwardEmailLog.forward_id)
-        .filter(ForwardEmail.gen_email_id == gen_email.id)
-        .order_by(ForwardEmailLog.id.desc())
+        Session.query(Contact, EmailLog)
+        .filter(Contact.id == EmailLog.contact_id)
+        .filter(Contact.alias_id == alias.id)
+        .order_by(EmailLog.id.desc())
         .limit(PAGE_LIMIT)
         .offset(page_id * PAGE_LIMIT)
     )
 
-    for fe, fel in q:
+    for contact, email_log in q:
         al = AliasLog(
-            website_email=fe.website_email,
-            website_from=fe.website_from,
-            alias=gen_email.email,
-            when=fel.created_at,
-            is_reply=fel.is_reply,
-            blocked=fel.blocked,
-            bounced=fel.bounced,
-            mailbox=mailbox,
+            website_email=contact.website_email,
+            reverse_alias=contact.website_send_to(),
+            alias=alias.email,
+            when=email_log.created_at,
+            is_reply=email_log.is_reply,
+            blocked=email_log.blocked,
+            bounced=email_log.bounced,
+            email_log=email_log,
+            contact=contact,
         )
         logs.append(al)
-    logs = sorted(logs, key=lambda l: l.when, reverse=True)
+    logs = sorted(logs, key=lambda log: log.when, reverse=True)
 
     return logs

app/dashboard/views/alias_transfer.py

@@ -0,0 +1,171 @@
+import base64
+import hmac
+import secrets
+
+import arrow
+from flask import render_template, redirect, url_for, flash, request
+from flask_login import login_required, current_user
+
+from app import config
+from app.alias_utils import transfer_alias
+from app.dashboard.base import dashboard_bp
+from app.dashboard.views.enter_sudo import sudo_required
+from app.db import Session
+from app.extensions import limiter
+from app.log import LOG
+from app.models import (
+    Alias,
+)
+from app.models import Mailbox
+from app.utils import CSRFValidationForm
+
+
+def hmac_alias_transfer_token(transfer_token: str) -> str:
+    alias_hmac = hmac.new(
+        config.ALIAS_TRANSFER_TOKEN_SECRET.encode("utf-8"),
+        transfer_token.encode("utf-8"),
+        "sha3_224",
+    )
+    return base64.urlsafe_b64encode(alias_hmac.digest()).decode("utf-8").rstrip("=")
+
+
+@dashboard_bp.route("/alias_transfer/send/<int:alias_id>/", methods=["GET", "POST"])
+@login_required
+@sudo_required
+def alias_transfer_send_route(alias_id):
+    alias = Alias.get(alias_id)
+    if not alias or alias.user_id != current_user.id:
+        flash("You cannot see this page", "warning")
+        return redirect(url_for("dashboard.index"))
+
+    if current_user.newsletter_alias_id == alias.id:
+        flash(
+            "This alias is currently used for receiving the newsletter and cannot be transferred",
+            "error",
+        )
+        return redirect(url_for("dashboard.index"))
+
+    alias_transfer_url = None
+    csrf_form = CSRFValidationForm()
+
+    if request.method == "POST":
+        if not csrf_form.validate():
+            flash("Invalid request", "warning")
+            return redirect(request.url)
+        # generate a new transfer_token
+        if request.form.get("form-name") == "create":
+            transfer_token = f"{alias.id}.{secrets.token_urlsafe(32)}"
+            alias.transfer_token = hmac_alias_transfer_token(transfer_token)
+            alias.transfer_token_expiration = arrow.utcnow().shift(hours=24)
+            Session.commit()
+            alias_transfer_url = (
+                config.URL
+                + "/dashboard/alias_transfer/receive"
+                + f"?token={transfer_token}"
+            )
+            flash("Share alias URL created", "success")
+        # request.form.get("form-name") == "remove"
+        else:
+            alias.transfer_token = None
+            alias.transfer_token_expiration = None
+            Session.commit()
+            alias_transfer_url = None
+            flash("Share URL deleted", "success")
+
+    return render_template(
+        "dashboard/alias_transfer_send.html",
+        alias=alias,
+        alias_transfer_url=alias_transfer_url,
+        link_active=alias.transfer_token_expiration is not None
+        and alias.transfer_token_expiration > arrow.utcnow(),
+        csrf_form=csrf_form,
+    )
+
+
+@dashboard_bp.route("/alias_transfer/receive", methods=["GET", "POST"])
+@limiter.limit("5/minute")
+@login_required
+def alias_transfer_receive_route():
+    """
+    URL has ?alias_id=signed_alias_id
+    """
+    token = request.args.get("token")
+    if not token:
+        flash("Invalid transfer token", "error")
+        return redirect(url_for("dashboard.index"))
+    hashed_token = hmac_alias_transfer_token(token)
+    # TODO: Don't allow unhashed tokens once all the tokens have been migrated to the new format
+    alias = Alias.get_by(transfer_token=token) or Alias.get_by(
+        transfer_token=hashed_token
+    )
+
+    if not alias:
+        flash("Invalid link", "error")
+        return redirect(url_for("dashboard.index"))
+
+    # TODO: Don't allow none once all the tokens have been migrated to the new format
+    if (
+        alias.transfer_token_expiration is not None
+        and alias.transfer_token_expiration < arrow.utcnow()
+    ):
+        flash("Expired link, please request a new one", "error")
+        return redirect(url_for("dashboard.index"))
+
+    # alias already belongs to this user
+    if alias.user_id == current_user.id:
+        flash("You already own this alias", "warning")
+        return redirect(url_for("dashboard.index"))
+
+    # check if user has not exceeded the alias quota
+    if not current_user.can_create_new_alias():
+        LOG.d("%s can't receive new alias", current_user)
+        flash(
+            "You have reached free plan limit, please upgrade to create new aliases",
+            "warning",
+        )
+        return redirect(url_for("dashboard.index"))
+
+    mailboxes = current_user.mailboxes()
+
+    if request.method == "POST":
+        mailbox_ids = request.form.getlist("mailbox_ids")
+        # check if mailbox is not tempered with
+        mailboxes = []
+        for mailbox_id in mailbox_ids:
+            mailbox = Mailbox.get(mailbox_id)
+            if (
+                not mailbox
+                or mailbox.user_id != current_user.id
+                or not mailbox.verified
+            ):
+                flash("Something went wrong, please retry", "warning")
+                return redirect(request.url)
+            mailboxes.append(mailbox)
+
+        if not mailboxes:
+            flash("You must select at least 1 mailbox", "warning")
+            return redirect(request.url)
+
+        LOG.d(
+            "transfer alias %s from %s to %s with %s with token %s",
+            alias,
+            alias.user,
+            current_user,
+            mailboxes,
+            token,
+        )
+        transfer_alias(alias, current_user, mailboxes)
+
+        # reset transfer token
+        alias.transfer_token = None
+        alias.transfer_token_expiration = None
+        Session.commit()
+
+        flash(f"You are now owner of {alias.email}", "success")
+        return redirect(url_for("dashboard.index", highlight_alias_id=alias.id))
+
+    return render_template(
+        "dashboard/alias_transfer_receive.html",
+        alias=alias,
+        mailboxes=mailboxes,
+    )

app/dashboard/views/api_key.py

@@ -3,23 +3,61 @@ from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators
 
+from app import config
 from app.dashboard.base import dashboard_bp
-from app.extensions import db
+from app.dashboard.views.enter_sudo import sudo_required
+from app.db import Session
+from app.extensions import limiter
 from app.models import ApiKey
+from app.utils import CSRFValidationForm
 
 
 class NewApiKeyForm(FlaskForm):
     name = StringField("Name", validators=[validators.DataRequired()])
 
 
+def clean_up_unused_or_old_api_keys(user_id: int):
+    total_keys = ApiKey.filter_by(user_id=user_id).count()
+    if total_keys <= config.MAX_API_KEYS:
+        return
+    # Remove oldest unused
+    for api_key in (
+        ApiKey.filter_by(user_id=user_id, last_used=None)
+        .order_by(ApiKey.created_at.asc())
+        .all()
+    ):
+        Session.delete(api_key)
+        total_keys -= 1
+        if total_keys <= config.MAX_API_KEYS:
+            return
+    # Clean up oldest used
+    for api_key in (
+        ApiKey.filter_by(user_id=user_id).order_by(ApiKey.last_used.asc()).all()
+    ):
+        Session.delete(api_key)
+        total_keys -= 1
+        if total_keys <= config.MAX_API_KEYS:
+            return
+
+
 @dashboard_bp.route("/api_key", methods=["GET", "POST"])
 @login_required
+@sudo_required
+@limiter.limit("10/hour")
 def api_key():
-    api_keys = ApiKey.query.filter(ApiKey.user_id == current_user.id).all()
+    api_keys = (
+        ApiKey.filter(ApiKey.user_id == current_user.id)
+        .order_by(ApiKey.created_at.desc())
+        .all()
+    )
 
+    csrf_form = CSRFValidationForm()
     new_api_key_form = NewApiKeyForm()
 
     if request.method == "POST":
+        if not csrf_form.validate():
+            flash("Invalid request", "warning")
+            return redirect(request.url)
         if request.form.get("form-name") == "delete":
             api_key_id = request.form.get("api-key-id")
 
@@ -34,23 +72,31 @@ def api_key():
 
             name = api_key.name
             ApiKey.delete(api_key_id)
-            db.session.commit()
+            Session.commit()
             flash(f"API Key {name} has been deleted", "success")
 
-            return redirect(url_for("dashboard.api_key"))
-
         elif request.form.get("form-name") == "create":
             if new_api_key_form.validate():
+                clean_up_unused_or_old_api_keys(current_user.id)
                 new_api_key = ApiKey.create(
                     name=new_api_key_form.name.data, user_id=current_user.id
                 )
-                db.session.commit()
-
+                Session.commit()
                 flash(f"New API Key {new_api_key.name} has been created", "success")
-                return redirect(url_for("dashboard.api_key"))
+                return render_template(
+                    "dashboard/new_api_key.html", api_key=new_api_key
+                )
+
+        elif request.form.get("form-name") == "delete-all":
+            ApiKey.delete_all(current_user.id)
+            Session.commit()
+            flash("All API Keys have been deleted", "success")
 
         return redirect(url_for("dashboard.api_key"))
 
     return render_template(
-        "dashboard/api_key.html", api_keys=api_keys, new_api_key_form=new_api_key_form
+        "dashboard/api_key.html",
+        api_keys=api_keys,
+        new_api_key_form=new_api_key_form,
+        csrf_form=csrf_form,
     )

app/dashboard/views/app.py

@@ -0,0 +1,47 @@
+from flask import render_template, request, flash, redirect
+from flask_login import login_required, current_user
+from sqlalchemy.orm import joinedload
+
+from app.dashboard.base import dashboard_bp
+from app.db import Session
+from app.models import (
+    ClientUser,
+)
+
+
+@dashboard_bp.route("/app", methods=["GET", "POST"])
+@login_required
+def app_route():
+    """
+    List of apps that user has used via the "Sign in with SimpleLogin"
+    """
+
+    client_users = (
+        ClientUser.filter_by(user_id=current_user.id)
+        .options(joinedload(ClientUser.client))
+        .options(joinedload(ClientUser.alias))
+        .all()
+    )
+
+    sorted(client_users, key=lambda cu: cu.client.name)
+
+    if request.method == "POST":
+        client_user_id = request.form.get("client-user-id")
+        client_user = ClientUser.get(client_user_id)
+        if not client_user or client_user.user_id != current_user.id:
+            flash(
+                "Unknown error, sorry for the inconvenience, refresh the page", "error"
+            )
+            return redirect(request.url)
+
+        client = client_user.client
+        ClientUser.delete(client_user_id)
+        Session.commit()
+
+        flash(f"Link with {client.name}  has been removed", "success")
+        return redirect(request.url)
+
+    return render_template(
+        "dashboard/app.html",
+        client_users=client_users,
+    )