From 6da48298a6fe66d21dfa2a2c85edb5fc1ec458e8 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Thu, 10 Sep 2020 20:05:25 +0200 Subject: [PATCH 1/3] Add BatchImport model --- app/models.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/app/models.py b/app/models.py index 27c464e4..ae3854c2 100644 --- a/app/models.py +++ b/app/models.py @@ -94,6 +94,9 @@ class File(db.Model, ModelMixin): def get_url(self, expires_in=3600): return s3.get_url(self.path, expires_in) + def __repr__(self): + return f"" + class EnumE(enum.Enum): @classmethod @@ -817,6 +820,13 @@ class Alias(db.Model, ModelMixin): db.Boolean, nullable=False, default=False, server_default="0" ) + # to know whether an alias is added using a batch import + batch_import_id = db.Column( + db.ForeignKey("batch_import.id", ondelete="SET NULL"), + nullable=True, + default=None, + ) + user = db.relationship(User) mailbox = db.relationship("Mailbox", lazy="joined") @@ -1742,3 +1752,19 @@ class Monitoring(db.Model, ModelMixin): incoming_queue = db.Column(db.Integer, nullable=False) active_queue = db.Column(db.Integer, nullable=False) deferred_queue = db.Column(db.Integer, nullable=False) + + +class BatchImport(db.Model, ModelMixin): + user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False) + file_id = db.Column(db.ForeignKey(File.id, ondelete="cascade"), nullable=False) + processed = db.Column(db.Boolean, nullable=False, default=False) + summary = db.Column(db.Text, nullable=True, default=None) + + file = db.relationship(File) + user = db.relationship(User) + + def nb_alias(self): + return Alias.query.filter_by(batch_import_id=self.id).count() + + def __repr__(self): + return f"" From f664243e42eb44fb057662da9376109618f09dcc Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Thu, 10 Sep 2020 20:14:55 +0200 Subject: [PATCH 2/3] add batch-import page --- app/config.py | 1 + app/dashboard/__init__.py | 1 + .../templates/dashboard/batch_import.html | 57 ++++++++++++++ .../templates/dashboard/setting.html | 29 ++++++-- app/dashboard/views/batch_import.py | 53 +++++++++++++ job_runner.py | 74 ++++++++++++++++++- static/batch_import_template.csv | 3 + 7 files changed, 208 insertions(+), 10 deletions(-) create mode 100644 app/dashboard/templates/dashboard/batch_import.html create mode 100644 app/dashboard/views/batch_import.py create mode 100644 static/batch_import_template.csv diff --git a/app/config.py b/app/config.py index 31763cc3..c794ed60 100644 --- a/app/config.py +++ b/app/config.py @@ -225,6 +225,7 @@ JOB_ONBOARDING_1 = "onboarding-1" JOB_ONBOARDING_2 = "onboarding-2" JOB_ONBOARDING_3 = "onboarding-3" JOB_ONBOARDING_4 = "onboarding-4" +JOB_BATCH_IMPORT = "batch-import" # for pagination PAGE_LIMIT = 20 diff --git a/app/dashboard/__init__.py b/app/dashboard/__init__.py index 95ac461b..8363aa5d 100644 --- a/app/dashboard/__init__.py +++ b/app/dashboard/__init__.py @@ -24,4 +24,5 @@ from .views import ( recovery_code, contact_detail, setup_done, + batch_import, ) diff --git a/app/dashboard/templates/dashboard/batch_import.html b/app/dashboard/templates/dashboard/batch_import.html new file mode 100644 index 00000000..bccda6c4 --- /dev/null +++ b/app/dashboard/templates/dashboard/batch_import.html @@ -0,0 +1,57 @@ +{% extends 'default.html' %} +{% set active_page = "setting" %} +{% block title %} + Alias Batch Import +{% endblock %} + +{% block default_content %} +
+
+

Alias Batch Import

+

+ The import can take several minutes. + Please come back to this page to verify the import status.
+ Only aliases created with your verified domains can be imported.
+ If an alias already exists, it won't be imported. +

+ + Download CSV Template + +
+ +
+ +
+ +
+ + {% if batch_imports %} +
+

Batch imports

+ + + + + + + + + + {% for batch_import in batch_imports %} + + + + + + {% endfor %} + +
UploadedNumber Alias ImportedStatus
{{ batch_import.created_at | dt }}{{ batch_import.nb_alias() }}{% if batch_import.processed %} Processed ✅ {% else %} Pending {% endif %}
+ {% endif %} + + +
+
+{% endblock %} diff --git a/app/dashboard/templates/dashboard/setting.html b/app/dashboard/templates/dashboard/setting.html index 7de883d4..11e630cf 100644 --- a/app/dashboard/templates/dashboard/setting.html +++ b/app/dashboard/templates/dashboard/setting.html @@ -8,11 +8,11 @@ {% block head %} {% endblock %} @@ -156,9 +156,9 @@ @@ -296,6 +296,19 @@ +
+
+
Import alias
+
+ You can import your aliases created on other platforms into SimpleLogin. +
+ + Batch Import + + +
+
+
Export Data
diff --git a/app/dashboard/views/batch_import.py b/app/dashboard/views/batch_import.py new file mode 100644 index 00000000..00e4a363 --- /dev/null +++ b/app/dashboard/views/batch_import.py @@ -0,0 +1,53 @@ +import arrow +from flask import render_template, flash, request, redirect, url_for +from flask_login import login_required, current_user +from flask_wtf import FlaskForm +from wtforms import StringField, validators + +from app import s3 +from app.config import JOB_BATCH_IMPORT +from app.dashboard.base import dashboard_bp +from app.extensions import db +from app.log import LOG +from app.models import CustomDomain, File, BatchImport, Job +from app.utils import random_string + + +@dashboard_bp.route("/batch_import", methods=["GET", "POST"]) +@login_required +def batch_import_route(): + # only for users who have custom domains + if not current_user.verified_custom_domains(): + flash("Alias batch import is only available for custom domains", "warning") + + batch_imports = BatchImport.query.filter_by(user_id=current_user.id).all() + + if request.method == "POST": + alias_file = request.files["alias-file"] + + file_path = random_string(20) + ".csv" + file = File.create(user_id=current_user.id, path=file_path) + s3.upload_from_bytesio(file_path, alias_file) + db.session.flush() + LOG.d("upload file %s to s3 at %s", file, file_path) + + bi = BatchImport.create(user_id=current_user.id, file_id=file.id) + db.session.flush() + LOG.debug("Add a batch import job %s for %s", bi, current_user) + + # Schedule batch import job + Job.create( + name=JOB_BATCH_IMPORT, + payload={"batch_import_id": bi.id}, + run_at=arrow.now(), + ) + db.session.commit() + + flash( + "The file has been uploaded successfully and the import will start shortly", + "success", + ) + + return redirect(url_for("dashboard.batch_import_route")) + + return render_template("dashboard/batch_import.html", batch_imports=batch_imports) diff --git a/job_runner.py b/job_runner.py index 70c580b5..c015731b 100644 --- a/job_runner.py +++ b/job_runner.py @@ -2,20 +2,36 @@ Run scheduled jobs. Not meant for running job at precise time (+- 1h) """ +import csv import time import arrow +import requests +from app import s3 from app.config import ( JOB_ONBOARDING_1, JOB_ONBOARDING_2, JOB_ONBOARDING_3, JOB_ONBOARDING_4, + JOB_BATCH_IMPORT, +) +from app.email_utils import ( + send_email, + render, + get_email_domain_part, ) -from app.email_utils import send_email, render from app.extensions import db from app.log import LOG -from app.models import User, Job +from app.models import ( + User, + Job, + BatchImport, + Alias, + DeletedAlias, + DomainDeletedAlias, + CustomDomain, +) from server import create_app @@ -71,6 +87,55 @@ def onboarding_mailbox(user): ) +def handle_batch_import(batch_import: BatchImport): + user = batch_import.user + + batch_import.processed = True + db.session.commit() + + LOG.debug("Start batch import for %s %s", batch_import, user) + file_url = s3.get_url(batch_import.file.path) + + LOG.d("Download file %s from %s", batch_import.file, file_url) + r = requests.get(file_url) + lines = [l.decode() for l in r.iter_lines()] + reader = csv.DictReader(lines) + + for row in reader: + full_alias = row["alias"].lower().strip().replace(" ", "") + note = row["note"] + + alias_domain = get_email_domain_part(full_alias) + custom_domain = CustomDomain.get_by(domain=alias_domain) + + if ( + not custom_domain + or not custom_domain.verified + or custom_domain.user_id != user.id + ): + LOG.debug("domain %s can't be used %s", alias_domain, user) + continue + + if ( + Alias.get_by(email=full_alias) + or DeletedAlias.get_by(email=full_alias) + or DomainDeletedAlias.get_by(email=full_alias) + ): + LOG.d("alias already used %s", full_alias) + continue + + alias = Alias.create( + user_id=user.id, + email=full_alias, + note=note, + mailbox_id=user.default_mailbox_id, + custom_domain_id=custom_domain.id, + batch_import_id=batch_import.id, + ) + db.session.commit() + LOG.d("Create %s", alias) + + if __name__ == "__main__": while True: # run a job 1h earlier or later is not a big deal ... @@ -129,6 +194,11 @@ if __name__ == "__main__": LOG.d("send onboarding pgp email to user %s", user) onboarding_pgp(user) + elif job.name == JOB_BATCH_IMPORT: + batch_import_id = job.payload.get("batch_import_id") + batch_import = BatchImport.get(batch_import_id) + handle_batch_import(batch_import) + else: LOG.exception("Unknown job name %s", job.name) diff --git a/static/batch_import_template.csv b/static/batch_import_template.csv new file mode 100644 index 00000000..c9af19f0 --- /dev/null +++ b/static/batch_import_template.csv @@ -0,0 +1,3 @@ +"alias","note" +"ebay@my-domain.com","Used on eBay" +"facebook@my-domain.com","Used on Facebook, Instagram." \ No newline at end of file From b92966b2c6434f8fa2dac4b9c368047e8c37022d Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Thu, 10 Sep 2020 20:15:21 +0200 Subject: [PATCH 3/3] sql migration --- .../versions/2020_091020_84471852b610_.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 migrations/versions/2020_091020_84471852b610_.py diff --git a/migrations/versions/2020_091020_84471852b610_.py b/migrations/versions/2020_091020_84471852b610_.py new file mode 100644 index 00000000..f2acc52b --- /dev/null +++ b/migrations/versions/2020_091020_84471852b610_.py @@ -0,0 +1,44 @@ +"""empty message + +Revision ID: 84471852b610 +Revises: b82bcad9accf +Create Date: 2020-09-10 20:15:10.956801 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '84471852b610' +down_revision = 'b82bcad9accf' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('batch_import', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('file_id', sa.Integer(), nullable=False), + sa.Column('processed', sa.Boolean(), nullable=False), + sa.Column('summary', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['file_id'], ['file.id'], ondelete='cascade'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'), + sa.PrimaryKeyConstraint('id') + ) + op.add_column('alias', sa.Column('batch_import_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'alias', 'batch_import', ['batch_import_id'], ['id'], ondelete='SET NULL') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'alias', type_='foreignkey') + op.drop_column('alias', 'batch_import_id') + op.drop_table('batch_import') + # ### end Alembic commands ###