From c6646d597198b53500de4e6f24e5911e6a8fb895 Mon Sep 17 00:00:00 2001 From: Sylvia van Os Date: Wed, 10 Mar 2021 22:35:04 +0100 Subject: [PATCH 1/6] Add export test --- app/api/views/export.py | 7 ++- static/batch_import_template.csv | 4 +- tests/api/test_import_export.py | 103 +++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 tests/api/test_import_export.py diff --git a/app/api/views/export.py b/app/api/views/export.py index 3557a116..82954e47 100644 --- a/app/api/views/export.py +++ b/app/api/views/export.py @@ -55,7 +55,12 @@ def export_aliases(): data = [["alias", "note", "enabled", "mailboxes"]] for alias in Alias.filter_by(user_id=user.id).all(): # type: Alias - mailboxes = " ".join([mailbox.email for mailbox in alias.mailboxes]) + # Always put the main mailbox first + # It is seen a primary while importing + alias_mailboxes = alias.mailboxes + alias_mailboxes.insert(0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox))) + + mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes]) data.append([alias.email, alias.note, alias.enabled, mailboxes]) si = StringIO() diff --git a/static/batch_import_template.csv b/static/batch_import_template.csv index a06b6439..ab107f49 100644 --- a/static/batch_import_template.csv +++ b/static/batch_import_template.csv @@ -1,3 +1,3 @@ "alias","note","mailboxes" -"ebay@my-domain.com","Used on eBay","destination@my-destionation-domain.com" -"facebook@my-domain.com","Used on Facebook, Instagram.","destination1@my-destionation-domain.com destination2@my-destination-domain.com" \ No newline at end of file +"ebay@my-domain.com","Used on eBay","destination@my-destination-domain.com" +"facebook@my-domain.com","Used on Facebook, Instagram.","destination1@my-destination-domain.com destination2@my-destination-domain.com" \ No newline at end of file diff --git a/tests/api/test_import_export.py b/tests/api/test_import_export.py new file mode 100644 index 00000000..28878609 --- /dev/null +++ b/tests/api/test_import_export.py @@ -0,0 +1,103 @@ +from flask import url_for + +from app import alias_utils +from app.extensions import db +from app.models import User, CustomDomain, Mailbox, Alias, AliasMailbox, ApiKey + +def test_export(flask_client): + # Create users + user1 = User.create( + email="a@b.c", + password="password", + name="Test User", + activated=True + ) + user2 = User.create( + email="x@y.z", + password="password", + name="Wrong user", + activated=True + ) + db.session.commit() + + # Remove onboarding aliases + for alias in Alias.filter_by(user_id=user1.id).all(): + alias_utils.delete_alias(alias, user1) + for alias in Alias.filter_by(user_id=user2.id).all(): + alias_utils.delete_alias(alias, user2) + db.session.commit() + + # Create domains + CustomDomain.create( + user_id=user1.id, + domain="my-destination-domain.com", + verified=True + ) + CustomDomain.create( + user_id=user2.id, + domain="bad-destionation-domain.com", + verified=True + ) + db.session.commit() + + # Create mailboxes + mailbox1 = Mailbox.create( + user_id=user1.id, + email="destination@my-destination-domain.com", + verified=True + ) + mailbox2 = Mailbox.create( + user_id=user1.id, + email="destination2@my-destination-domain.com", + verified=True + ) + badmailbox1 = Mailbox.create( + user_id=user2.id, + email="baddestination@bad-destination-domain.com", + verified=True + ) + db.session.commit() + + # Create aliases + alias1 = Alias.create( + user_id=user1.id, + email="ebay@my-domain.com", + note="Used on eBay", + mailbox_id=mailbox1.id + ) + alias2 = Alias.create( + user_id=user1.id, + email="facebook@my-domain.com", + note="Used on Facebook, Instagram.", + mailbox_id=mailbox1.id + ) + alias3 = Alias.create( + user_id=user2.id, + email="notmine@my-domain.com", + note="Should not appear", + mailbox_id=badmailbox1.id + ) + db.session.commit() + + # Add second mailbox to an alias + alias_mailbox = AliasMailbox.create( + alias_id=alias2.id, + mailbox_id=mailbox2.id, + ) + db.session.commit() + + # Export + # Create api_key + api_key = ApiKey.create(user1.id, "for test") + db.session.commit() + + # <<< without hostname >>> + r = flask_client.get( + url_for("api.export_aliases"), headers={"Authentication": api_key.code} + ) + assert r.status_code == 200 + assert r.mimetype == "text/csv" + assert r.data == """alias,note,enabled,mailboxes +ebay@my-domain.com,Used on eBay,True,destination@my-destination-domain.com +facebook@my-domain.com,"Used on Facebook, Instagram.",True,destination@my-destination-domain.com destination2@my-destination-domain.com +""".replace("\n", "\r\n").encode() \ No newline at end of file From 44ae20816a5099894bfe4628aec05f87643cb678 Mon Sep 17 00:00:00 2001 From: Sylvia van Os Date: Wed, 10 Mar 2021 23:08:33 +0100 Subject: [PATCH 2/6] WIP: Import test --- app/import_utils.py | 84 +++++++++++++++++++++++++++++++++ job_runner.py | 79 +------------------------------ tests/api/test_import_export.py | 39 +++++++++++++-- 3 files changed, 121 insertions(+), 81 deletions(-) create mode 100644 app/import_utils.py diff --git a/app/import_utils.py b/app/import_utils.py new file mode 100644 index 00000000..69c4cb24 --- /dev/null +++ b/app/import_utils.py @@ -0,0 +1,84 @@ +from .log import LOG + +import requests + +from app import s3 +from app.extensions import db +from app.models import BatchImport + +def handle_batch_import(batch_import: BatchImport): + user = batch_import.user + + batch_import.processed = True + db.session.commit() + + LOG.debug("Start batch import for %s %s", batch_import, user) + file_url = s3.get_url(batch_import.file.path) + + LOG.d("Download file %s from %s", batch_import.file, file_url) + r = requests.get(file_url) + lines = [line.decode() for line in r.iter_lines()] + reader = csv.DictReader(lines) + + for row in reader: + try: + full_alias = sanitize_email(row["alias"]) + note = row["note"] + except KeyError: + LOG.warning("Cannot parse row %s", row) + continue + + alias_domain = get_email_domain_part(full_alias) + custom_domain = CustomDomain.get_by(domain=alias_domain) + + if ( + not custom_domain + or not custom_domain.verified + or custom_domain.user_id != user.id + ): + LOG.debug("domain %s can't be used %s", alias_domain, user) + continue + + if ( + Alias.get_by(email=full_alias) + or DeletedAlias.get_by(email=full_alias) + or DomainDeletedAlias.get_by(email=full_alias) + ): + LOG.d("alias already used %s", full_alias) + continue + + mailboxes = [] + + if "mailboxes" in row: + for mailbox_email in row["mailboxes"].split(): + mailbox_email = sanitize_email(mailbox_email) + mailbox = Mailbox.get_by(email=mailbox_email) + + if not mailbox or not mailbox.verified or mailbox.user_id != user.id: + LOG.d("mailbox %s can't be used %s", mailbox, user) + continue + + mailboxes.append(mailbox.id) + + if len(mailboxes) == 0: + mailboxes = [user.default_mailbox_id] + + alias = Alias.create( + user_id=user.id, + email=full_alias, + note=note, + mailbox_id=mailboxes[0], + custom_domain_id=custom_domain.id, + batch_import_id=batch_import.id, + ) + db.session.commit() + db.session.flush() + LOG.d("Create %s", alias) + + for i in range(1, len(mailboxes)): + alias_mailbox = AliasMailbox.create( + alias_id=alias.id, + mailbox_id=mailboxes[i], + ) + db.session.commit() + LOG.d("Create %s", alias_mailbox) \ No newline at end of file diff --git a/job_runner.py b/job_runner.py index b426332f..1ad6b8a2 100644 --- a/job_runner.py +++ b/job_runner.py @@ -20,6 +20,7 @@ from app.email_utils import ( render, get_email_domain_part, ) +from app.import_utils import handle_batch_import from app.utils import sanitize_email from app.extensions import db from app.log import LOG @@ -113,84 +114,6 @@ def onboarding_mailbox(user): ) -def handle_batch_import(batch_import: BatchImport): - user = batch_import.user - - batch_import.processed = True - db.session.commit() - - LOG.debug("Start batch import for %s %s", batch_import, user) - file_url = s3.get_url(batch_import.file.path) - - LOG.d("Download file %s from %s", batch_import.file, file_url) - r = requests.get(file_url) - lines = [line.decode() for line in r.iter_lines()] - reader = csv.DictReader(lines) - - for row in reader: - try: - full_alias = sanitize_email(row["alias"]) - note = row["note"] - except KeyError: - LOG.warning("Cannot parse row %s", row) - continue - - alias_domain = get_email_domain_part(full_alias) - custom_domain = CustomDomain.get_by(domain=alias_domain) - - if ( - not custom_domain - or not custom_domain.verified - or custom_domain.user_id != user.id - ): - LOG.debug("domain %s can't be used %s", alias_domain, user) - continue - - if ( - Alias.get_by(email=full_alias) - or DeletedAlias.get_by(email=full_alias) - or DomainDeletedAlias.get_by(email=full_alias) - ): - LOG.d("alias already used %s", full_alias) - continue - - mailboxes = [] - - if "mailboxes" in row: - for mailbox_email in row["mailboxes"].split(): - mailbox_email = sanitize_email(mailbox_email) - mailbox = Mailbox.get_by(email=mailbox_email) - - if not mailbox or not mailbox.verified or mailbox.user_id != user.id: - LOG.d("mailbox %s can't be used %s", mailbox, user) - continue - - mailboxes.append(mailbox.id) - - if len(mailboxes) == 0: - mailboxes = [user.default_mailbox_id] - - alias = Alias.create( - user_id=user.id, - email=full_alias, - note=note, - mailbox_id=mailboxes[0], - custom_domain_id=custom_domain.id, - batch_import_id=batch_import.id, - ) - db.session.commit() - db.session.flush() - LOG.d("Create %s", alias) - - for i in range(1, len(mailboxes)): - alias_mailbox = AliasMailbox.create( - alias_id=alias.id, - mailbox_id=mailboxes[i], - ) - db.session.commit() - LOG.d("Create %s", alias_mailbox) - - if __name__ == "__main__": while True: # run a job 1h earlier or later is not a big deal ... diff --git a/tests/api/test_import_export.py b/tests/api/test_import_export.py index 28878609..67e55e48 100644 --- a/tests/api/test_import_export.py +++ b/tests/api/test_import_export.py @@ -1,8 +1,13 @@ +from io import BytesIO +from os import path + from flask import url_for -from app import alias_utils +from app import alias_utils, s3 from app.extensions import db -from app.models import User, CustomDomain, Mailbox, Alias, AliasMailbox, ApiKey +from app.models import User, CustomDomain, Mailbox, Alias, AliasMailbox, ApiKey, File, BatchImport +from app.import_utils import handle_batch_import +from app.utils import random_string def test_export(flask_client): # Create users @@ -100,4 +105,32 @@ def test_export(flask_client): assert r.data == """alias,note,enabled,mailboxes ebay@my-domain.com,Used on eBay,True,destination@my-destination-domain.com facebook@my-domain.com,"Used on Facebook, Instagram.",True,destination@my-destination-domain.com destination2@my-destination-domain.com -""".replace("\n", "\r\n").encode() \ No newline at end of file +""".replace("\n", "\r\n").encode() + +def test_import_no_mailboxes(flask_client): + # Create user + user = User.create( + email="a@b.c", + password="password", + name="Test User", + activated=True + ) + db.session.commit() + + alias_file = BytesIO(b"""alias,note,enabled +ebay@my-domain.com,Used on eBay,True +facebook@my-domain.com,"Used on Facebook, Instagram.",True +""") + + file_path = random_string(20) + ".csv" + file = File.create(user_id=user.id, path=file_path) + s3.upload_from_bytesio(file_path, alias_file) + db.session.flush() + + batch_import = BatchImport.create( + user_id=user.id, + file_id=file.id + ) + db.session.commit() + + handle_batch_import(batch_import) \ No newline at end of file From dcfd63eb0f0fb1ba6bc7d8b1235ec8bfcf5a7b19 Mon Sep 17 00:00:00 2001 From: Sylvia van Os Date: Sat, 13 Mar 2021 00:13:33 +0100 Subject: [PATCH 3/6] Working import tests --- app/import_utils.py | 9 +++- app/utils.py | 2 +- tests/api/test_import_export.py | 83 +++++++++++++++++++-------------- tests/utils.py | 2 +- 4 files changed, 58 insertions(+), 38 deletions(-) diff --git a/app/import_utils.py b/app/import_utils.py index 69c4cb24..4d6a6f61 100644 --- a/app/import_utils.py +++ b/app/import_utils.py @@ -1,10 +1,13 @@ from .log import LOG +import csv import requests from app import s3 +from app.email_utils import get_email_domain_part from app.extensions import db -from app.models import BatchImport +from app.models import Alias, BatchImport, CustomDomain, DeletedAlias, DomainDeletedAlias, User +from app.utils import sanitize_email def handle_batch_import(batch_import: BatchImport): user = batch_import.user @@ -18,6 +21,10 @@ def handle_batch_import(batch_import: BatchImport): LOG.d("Download file %s from %s", batch_import.file, file_url) r = requests.get(file_url) lines = [line.decode() for line in r.iter_lines()] + + import_from_csv(user, lines) + +def import_from_csv(batch_import: BatchImport, user: User, lines): reader = csv.DictReader(lines) for row in reader: diff --git a/app/utils.py b/app/utils.py index 61fd2ee7..789b91eb 100644 --- a/app/utils.py +++ b/app/utils.py @@ -64,4 +64,4 @@ def encode_url(url): def sanitize_email(email_address: str) -> str: if email_address: return email_address.lower().strip().replace(" ", "") - return email_address + return email_address \ No newline at end of file diff --git a/tests/api/test_import_export.py b/tests/api/test_import_export.py index 67e55e48..99dff079 100644 --- a/tests/api/test_import_export.py +++ b/tests/api/test_import_export.py @@ -1,22 +1,15 @@ -from io import BytesIO -from os import path - from flask import url_for -from app import alias_utils, s3 +from app import alias_utils from app.extensions import db from app.models import User, CustomDomain, Mailbox, Alias, AliasMailbox, ApiKey, File, BatchImport -from app.import_utils import handle_batch_import +from app.import_utils import import_from_csv from app.utils import random_string +from tests.utils import login def test_export(flask_client): # Create users - user1 = User.create( - email="a@b.c", - password="password", - name="Test User", - activated=True - ) + user1 = login(flask_client) user2 = User.create( email="x@y.z", password="password", @@ -92,13 +85,8 @@ def test_export(flask_client): db.session.commit() # Export - # Create api_key - api_key = ApiKey.create(user1.id, "for test") - db.session.commit() - - # <<< without hostname >>> r = flask_client.get( - url_for("api.export_aliases"), headers={"Authentication": api_key.code} + url_for("api.export_aliases") ) assert r.status_code == 200 assert r.mimetype == "text/csv" @@ -107,30 +95,55 @@ ebay@my-domain.com,Used on eBay,True,destination@my-destination-domain.com facebook@my-domain.com,"Used on Facebook, Instagram.",True,destination@my-destination-domain.com destination2@my-destination-domain.com """.replace("\n", "\r\n").encode() -def test_import_no_mailboxes(flask_client): +def test_import_no_mailboxes_no_domains(flask_client): # Create user - user = User.create( - email="a@b.c", - password="password", - name="Test User", - activated=True - ) - db.session.commit() + user = login(flask_client) - alias_file = BytesIO(b"""alias,note,enabled -ebay@my-domain.com,Used on eBay,True -facebook@my-domain.com,"Used on Facebook, Instagram.",True -""") + # Check start state + assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias - file_path = random_string(20) + ".csv" - file = File.create(user_id=user.id, path=file_path) - s3.upload_from_bytesio(file_path, alias_file) - db.session.flush() + alias_data = [ + 'alias,note,enabled', + 'ebay@my-domain.com,Used on eBay,True', + 'facebook@my-domain.com,"Used on Facebook, Instagram.",True' + ] batch_import = BatchImport.create( user_id=user.id, - file_id=file.id + file_id=0 + ) + + import_from_csv(batch_import, user, alias_data) + + # Should have failed to import anything new because my-domain.com isn't registered + assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # +0 + +def test_import_no_mailboxes(flask_client): + # Create user + user = login(flask_client) + + # Check start state + assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias + + # Create domain + CustomDomain.create( + user_id=user.id, + domain="my-domain.com", + verified=True ) db.session.commit() - handle_batch_import(batch_import) \ No newline at end of file + alias_data = [ + 'alias,note,enabled', + 'ebay@my-domain.com,Used on eBay,True', + 'facebook@my-domain.com,"Used on Facebook, Instagram.",True' + ] + + batch_import = BatchImport.create( + user_id=user.id, + file_id=0 + ) + + import_from_csv(batch_import, user, alias_data) + + assert(len(Alias.filter_by(user_id=user.id).all()) == 3) # +2 diff --git a/tests/utils.py b/tests/utils.py index 5de1a4a4..fb5994bf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -40,4 +40,4 @@ def create_user(flask_client) -> User: def pretty(d): """pretty print as json""" - print(json.dumps(d, indent=2)) + print(json.dumps(d, indent=2)) \ No newline at end of file From 8800c29526fd2691c12db94fa25f6cfb316adbba Mon Sep 17 00:00:00 2001 From: Sylvia van Os Date: Sat, 13 Mar 2021 15:36:25 +0100 Subject: [PATCH 4/6] Complete import tests --- app/import_utils.py | 2 +- tests/api/test_import_export.py | 103 ++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/app/import_utils.py b/app/import_utils.py index 4d6a6f61..33a90319 100644 --- a/app/import_utils.py +++ b/app/import_utils.py @@ -6,7 +6,7 @@ import requests from app import s3 from app.email_utils import get_email_domain_part from app.extensions import db -from app.models import Alias, BatchImport, CustomDomain, DeletedAlias, DomainDeletedAlias, User +from app.models import Alias, AliasMailbox, BatchImport, CustomDomain, DeletedAlias, DomainDeletedAlias, Mailbox, User from app.utils import sanitize_email def handle_batch_import(batch_import: BatchImport): diff --git a/tests/api/test_import_export.py b/tests/api/test_import_export.py index 99dff079..38b80bb1 100644 --- a/tests/api/test_import_export.py +++ b/tests/api/test_import_export.py @@ -103,9 +103,9 @@ def test_import_no_mailboxes_no_domains(flask_client): assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias alias_data = [ - 'alias,note,enabled', - 'ebay@my-domain.com,Used on eBay,True', - 'facebook@my-domain.com,"Used on Facebook, Instagram.",True' + 'alias,note', + 'ebay@my-domain.com,Used on eBay', + 'facebook@my-domain.com,"Used on Facebook, Instagram."' ] batch_import = BatchImport.create( @@ -134,9 +134,9 @@ def test_import_no_mailboxes(flask_client): db.session.commit() alias_data = [ - 'alias,note,enabled', - 'ebay@my-domain.com,Used on eBay,True', - 'facebook@my-domain.com,"Used on Facebook, Instagram.",True' + 'alias,note', + 'ebay@my-domain.com,Used on eBay', + 'facebook@my-domain.com,"Used on Facebook, Instagram."' ] batch_import = BatchImport.create( @@ -147,3 +147,94 @@ def test_import_no_mailboxes(flask_client): import_from_csv(batch_import, user, alias_data) assert(len(Alias.filter_by(user_id=user.id).all()) == 3) # +2 + +def test_import_no_domains(flask_client): + # Create user + user = login(flask_client) + + # Check start state + assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias + + alias_data = [ + 'alias,note,mailboxes', + 'ebay@my-domain.com,Used on eBay,destination@my-destination-domain.com', + 'facebook@my-domain.com,"Used on Facebook, Instagram.",destination1@my-destination-domain.com destination2@my-destination-domain.com' + ] + + batch_import = BatchImport.create( + user_id=user.id, + file_id=0 + ) + + import_from_csv(batch_import, user, alias_data) + + # Should have failed to import anything new because my-domain.com isn't registered + assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # +0 + +def test_import(flask_client): + # Create user + user = login(flask_client) + + # Check start state + assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias + + # Create domains + domain1 = CustomDomain.create( + user_id=user.id, + domain="my-domain.com", + verified=True + ) + domain2 = CustomDomain.create( + user_id=user.id, + domain="my-destination-domain.com", + verified=True + ) + db.session.commit() + + # Create mailboxes + mailbox1 = Mailbox.create( + user_id=user.id, + email="destination@my-destination-domain.com", + verified=True + ) + mailbox2 = Mailbox.create( + user_id=user.id, + email="destination2@my-destination-domain.com", + verified=True + ) + db.session.commit() + + alias_data = [ + 'alias,note,mailboxes', + 'ebay@my-domain.com,Used on eBay,destination@my-destination-domain.com', + 'facebook@my-domain.com,"Used on Facebook, Instagram.",destination@my-destination-domain.com destination2@my-destination-domain.com' + ] + + batch_import = BatchImport.create( + user_id=user.id, + file_id=0 + ) + + import_from_csv(batch_import, user, alias_data) + + aliases = Alias.filter_by(user_id=user.id).all() + assert(len(aliases) == 3) # +2 + + # aliases[0] is the onboarding alias, skip it + + # eBay alias + assert(aliases[1].email == "ebay@my-domain.com") + assert(len(aliases[1].mailboxes) == 1) + # First one should be primary + assert(aliases[1].mailbox_id == mailbox1.id) + # Others are sorted + assert(aliases[1].mailboxes[0] == mailbox1) + + # Facebook alias + assert(aliases[2].email == "facebook@my-domain.com") + assert(len(aliases[2].mailboxes) == 2) + # First one should be primary + assert(aliases[2].mailbox_id == mailbox1.id) + # Others are sorted + assert(aliases[2].mailboxes[0] == mailbox2) + assert(aliases[2].mailboxes[1] == mailbox1) \ No newline at end of file From cfe889f7b9c993d2a8bb49d4ea37178a4e572e5a Mon Sep 17 00:00:00 2001 From: Sylvia van Os Date: Sat, 13 Mar 2021 15:37:28 +0100 Subject: [PATCH 5/6] Format with black --- app/api/views/export.py | 4 +- app/import_utils.py | 15 ++- app/utils.py | 2 +- tests/api/test_import_export.py | 158 ++++++++++++++------------------ tests/utils.py | 2 +- 5 files changed, 88 insertions(+), 93 deletions(-) diff --git a/app/api/views/export.py b/app/api/views/export.py index 82954e47..ba0fe211 100644 --- a/app/api/views/export.py +++ b/app/api/views/export.py @@ -58,7 +58,9 @@ def export_aliases(): # Always put the main mailbox first # It is seen a primary while importing alias_mailboxes = alias.mailboxes - alias_mailboxes.insert(0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox))) + alias_mailboxes.insert( + 0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox)) + ) mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes]) data.append([alias.email, alias.note, alias.enabled, mailboxes]) diff --git a/app/import_utils.py b/app/import_utils.py index 33a90319..c87b0f6b 100644 --- a/app/import_utils.py +++ b/app/import_utils.py @@ -6,9 +6,19 @@ import requests from app import s3 from app.email_utils import get_email_domain_part from app.extensions import db -from app.models import Alias, AliasMailbox, BatchImport, CustomDomain, DeletedAlias, DomainDeletedAlias, Mailbox, User +from app.models import ( + Alias, + AliasMailbox, + BatchImport, + CustomDomain, + DeletedAlias, + DomainDeletedAlias, + Mailbox, + User, +) from app.utils import sanitize_email + def handle_batch_import(batch_import: BatchImport): user = batch_import.user @@ -24,6 +34,7 @@ def handle_batch_import(batch_import: BatchImport): import_from_csv(user, lines) + def import_from_csv(batch_import: BatchImport, user: User, lines): reader = csv.DictReader(lines) @@ -88,4 +99,4 @@ def import_from_csv(batch_import: BatchImport, user: User, lines): mailbox_id=mailboxes[i], ) db.session.commit() - LOG.d("Create %s", alias_mailbox) \ No newline at end of file + LOG.d("Create %s", alias_mailbox) diff --git a/app/utils.py b/app/utils.py index 789b91eb..61fd2ee7 100644 --- a/app/utils.py +++ b/app/utils.py @@ -64,4 +64,4 @@ def encode_url(url): def sanitize_email(email_address: str) -> str: if email_address: return email_address.lower().strip().replace(" ", "") - return email_address \ No newline at end of file + return email_address diff --git a/tests/api/test_import_export.py b/tests/api/test_import_export.py index 38b80bb1..c37de454 100644 --- a/tests/api/test_import_export.py +++ b/tests/api/test_import_export.py @@ -2,19 +2,26 @@ from flask import url_for from app import alias_utils from app.extensions import db -from app.models import User, CustomDomain, Mailbox, Alias, AliasMailbox, ApiKey, File, BatchImport +from app.models import ( + User, + CustomDomain, + Mailbox, + Alias, + AliasMailbox, + ApiKey, + File, + BatchImport, +) from app.import_utils import import_from_csv from app.utils import random_string from tests.utils import login + def test_export(flask_client): # Create users user1 = login(flask_client) user2 = User.create( - email="x@y.z", - password="password", - name="Wrong user", - activated=True + email="x@y.z", password="password", name="Wrong user", activated=True ) db.session.commit() @@ -27,32 +34,24 @@ def test_export(flask_client): # Create domains CustomDomain.create( - user_id=user1.id, - domain="my-destination-domain.com", - verified=True + user_id=user1.id, domain="my-destination-domain.com", verified=True ) CustomDomain.create( - user_id=user2.id, - domain="bad-destionation-domain.com", - verified=True + user_id=user2.id, domain="bad-destionation-domain.com", verified=True ) db.session.commit() # Create mailboxes mailbox1 = Mailbox.create( - user_id=user1.id, - email="destination@my-destination-domain.com", - verified=True + user_id=user1.id, email="destination@my-destination-domain.com", verified=True ) mailbox2 = Mailbox.create( - user_id=user1.id, - email="destination2@my-destination-domain.com", - verified=True + user_id=user1.id, email="destination2@my-destination-domain.com", verified=True ) badmailbox1 = Mailbox.create( user_id=user2.id, email="baddestination@bad-destination-domain.com", - verified=True + verified=True, ) db.session.commit() @@ -61,19 +60,19 @@ def test_export(flask_client): user_id=user1.id, email="ebay@my-domain.com", note="Used on eBay", - mailbox_id=mailbox1.id + mailbox_id=mailbox1.id, ) alias2 = Alias.create( user_id=user1.id, email="facebook@my-domain.com", note="Used on Facebook, Instagram.", - mailbox_id=mailbox1.id + mailbox_id=mailbox1.id, ) alias3 = Alias.create( user_id=user2.id, email="notmine@my-domain.com", note="Should not appear", - mailbox_id=badmailbox1.id + mailbox_id=badmailbox1.id, ) db.session.commit() @@ -85,156 +84,139 @@ def test_export(flask_client): db.session.commit() # Export - r = flask_client.get( - url_for("api.export_aliases") - ) + r = flask_client.get(url_for("api.export_aliases")) assert r.status_code == 200 assert r.mimetype == "text/csv" - assert r.data == """alias,note,enabled,mailboxes + assert ( + r.data + == """alias,note,enabled,mailboxes ebay@my-domain.com,Used on eBay,True,destination@my-destination-domain.com facebook@my-domain.com,"Used on Facebook, Instagram.",True,destination@my-destination-domain.com destination2@my-destination-domain.com -""".replace("\n", "\r\n").encode() +""".replace( + "\n", "\r\n" + ).encode() + ) + def test_import_no_mailboxes_no_domains(flask_client): # Create user user = login(flask_client) # Check start state - assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias + assert len(Alias.filter_by(user_id=user.id).all()) == 1 # Onboarding alias alias_data = [ - 'alias,note', - 'ebay@my-domain.com,Used on eBay', - 'facebook@my-domain.com,"Used on Facebook, Instagram."' + "alias,note", + "ebay@my-domain.com,Used on eBay", + 'facebook@my-domain.com,"Used on Facebook, Instagram."', ] - batch_import = BatchImport.create( - user_id=user.id, - file_id=0 - ) + batch_import = BatchImport.create(user_id=user.id, file_id=0) import_from_csv(batch_import, user, alias_data) # Should have failed to import anything new because my-domain.com isn't registered - assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # +0 + assert len(Alias.filter_by(user_id=user.id).all()) == 1 # +0 + def test_import_no_mailboxes(flask_client): # Create user user = login(flask_client) # Check start state - assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias + assert len(Alias.filter_by(user_id=user.id).all()) == 1 # Onboarding alias # Create domain - CustomDomain.create( - user_id=user.id, - domain="my-domain.com", - verified=True - ) + CustomDomain.create(user_id=user.id, domain="my-domain.com", verified=True) db.session.commit() alias_data = [ - 'alias,note', - 'ebay@my-domain.com,Used on eBay', - 'facebook@my-domain.com,"Used on Facebook, Instagram."' + "alias,note", + "ebay@my-domain.com,Used on eBay", + 'facebook@my-domain.com,"Used on Facebook, Instagram."', ] - batch_import = BatchImport.create( - user_id=user.id, - file_id=0 - ) + batch_import = BatchImport.create(user_id=user.id, file_id=0) import_from_csv(batch_import, user, alias_data) - assert(len(Alias.filter_by(user_id=user.id).all()) == 3) # +2 + assert len(Alias.filter_by(user_id=user.id).all()) == 3 # +2 + def test_import_no_domains(flask_client): # Create user user = login(flask_client) # Check start state - assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias + assert len(Alias.filter_by(user_id=user.id).all()) == 1 # Onboarding alias alias_data = [ - 'alias,note,mailboxes', - 'ebay@my-domain.com,Used on eBay,destination@my-destination-domain.com', - 'facebook@my-domain.com,"Used on Facebook, Instagram.",destination1@my-destination-domain.com destination2@my-destination-domain.com' + "alias,note,mailboxes", + "ebay@my-domain.com,Used on eBay,destination@my-destination-domain.com", + 'facebook@my-domain.com,"Used on Facebook, Instagram.",destination1@my-destination-domain.com destination2@my-destination-domain.com', ] - batch_import = BatchImport.create( - user_id=user.id, - file_id=0 - ) + batch_import = BatchImport.create(user_id=user.id, file_id=0) import_from_csv(batch_import, user, alias_data) # Should have failed to import anything new because my-domain.com isn't registered - assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # +0 + assert len(Alias.filter_by(user_id=user.id).all()) == 1 # +0 + def test_import(flask_client): # Create user user = login(flask_client) # Check start state - assert(len(Alias.filter_by(user_id=user.id).all()) == 1) # Onboarding alias + assert len(Alias.filter_by(user_id=user.id).all()) == 1 # Onboarding alias # Create domains domain1 = CustomDomain.create( - user_id=user.id, - domain="my-domain.com", - verified=True + user_id=user.id, domain="my-domain.com", verified=True ) domain2 = CustomDomain.create( - user_id=user.id, - domain="my-destination-domain.com", - verified=True + user_id=user.id, domain="my-destination-domain.com", verified=True ) db.session.commit() # Create mailboxes mailbox1 = Mailbox.create( - user_id=user.id, - email="destination@my-destination-domain.com", - verified=True + user_id=user.id, email="destination@my-destination-domain.com", verified=True ) mailbox2 = Mailbox.create( - user_id=user.id, - email="destination2@my-destination-domain.com", - verified=True + user_id=user.id, email="destination2@my-destination-domain.com", verified=True ) db.session.commit() alias_data = [ - 'alias,note,mailboxes', - 'ebay@my-domain.com,Used on eBay,destination@my-destination-domain.com', - 'facebook@my-domain.com,"Used on Facebook, Instagram.",destination@my-destination-domain.com destination2@my-destination-domain.com' + "alias,note,mailboxes", + "ebay@my-domain.com,Used on eBay,destination@my-destination-domain.com", + 'facebook@my-domain.com,"Used on Facebook, Instagram.",destination@my-destination-domain.com destination2@my-destination-domain.com', ] - batch_import = BatchImport.create( - user_id=user.id, - file_id=0 - ) + batch_import = BatchImport.create(user_id=user.id, file_id=0) import_from_csv(batch_import, user, alias_data) aliases = Alias.filter_by(user_id=user.id).all() - assert(len(aliases) == 3) # +2 + assert len(aliases) == 3 # +2 # aliases[0] is the onboarding alias, skip it # eBay alias - assert(aliases[1].email == "ebay@my-domain.com") - assert(len(aliases[1].mailboxes) == 1) + assert aliases[1].email == "ebay@my-domain.com" + assert len(aliases[1].mailboxes) == 1 # First one should be primary - assert(aliases[1].mailbox_id == mailbox1.id) + assert aliases[1].mailbox_id == mailbox1.id # Others are sorted - assert(aliases[1].mailboxes[0] == mailbox1) + assert aliases[1].mailboxes[0] == mailbox1 # Facebook alias - assert(aliases[2].email == "facebook@my-domain.com") - assert(len(aliases[2].mailboxes) == 2) + assert aliases[2].email == "facebook@my-domain.com" + assert len(aliases[2].mailboxes) == 2 # First one should be primary - assert(aliases[2].mailbox_id == mailbox1.id) + assert aliases[2].mailbox_id == mailbox1.id # Others are sorted - assert(aliases[2].mailboxes[0] == mailbox2) - assert(aliases[2].mailboxes[1] == mailbox1) \ No newline at end of file + assert aliases[2].mailboxes[0] == mailbox2 + assert aliases[2].mailboxes[1] == mailbox1 diff --git a/tests/utils.py b/tests/utils.py index fb5994bf..5de1a4a4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -40,4 +40,4 @@ def create_user(flask_client) -> User: def pretty(d): """pretty print as json""" - print(json.dumps(d, indent=2)) \ No newline at end of file + print(json.dumps(d, indent=2)) From e02d95216f3e2afc47dc42d02cece801b2b8ad4e Mon Sep 17 00:00:00 2001 From: Sylvia van Os Date: Sat, 13 Mar 2021 15:46:12 +0100 Subject: [PATCH 6/6] Remove unused imports and var assignments --- job_runner.py | 17 +---------------- tests/api/test_import_export.py | 15 +++++---------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/job_runner.py b/job_runner.py index 1ad6b8a2..c0ee9651 100644 --- a/job_runner.py +++ b/job_runner.py @@ -2,13 +2,10 @@ Run scheduled jobs. Not meant for running job at precise time (+- 1h) """ -import csv import time import arrow -import requests -from app import s3 from app.config import ( JOB_ONBOARDING_1, JOB_ONBOARDING_2, @@ -18,23 +15,11 @@ from app.config import ( from app.email_utils import ( send_email, render, - get_email_domain_part, ) from app.import_utils import handle_batch_import -from app.utils import sanitize_email from app.extensions import db from app.log import LOG -from app.models import ( - User, - Job, - BatchImport, - Alias, - DeletedAlias, - DomainDeletedAlias, - CustomDomain, - Mailbox, - AliasMailbox, -) +from app.models import User, Job, BatchImport from server import create_app diff --git a/tests/api/test_import_export.py b/tests/api/test_import_export.py index c37de454..c5ee865d 100644 --- a/tests/api/test_import_export.py +++ b/tests/api/test_import_export.py @@ -8,12 +8,9 @@ from app.models import ( Mailbox, Alias, AliasMailbox, - ApiKey, - File, BatchImport, ) from app.import_utils import import_from_csv -from app.utils import random_string from tests.utils import login @@ -56,7 +53,7 @@ def test_export(flask_client): db.session.commit() # Create aliases - alias1 = Alias.create( + Alias.create( user_id=user1.id, email="ebay@my-domain.com", note="Used on eBay", @@ -68,7 +65,7 @@ def test_export(flask_client): note="Used on Facebook, Instagram.", mailbox_id=mailbox1.id, ) - alias3 = Alias.create( + Alias.create( user_id=user2.id, email="notmine@my-domain.com", note="Should not appear", @@ -77,7 +74,7 @@ def test_export(flask_client): db.session.commit() # Add second mailbox to an alias - alias_mailbox = AliasMailbox.create( + AliasMailbox.create( alias_id=alias2.id, mailbox_id=mailbox2.id, ) @@ -172,10 +169,8 @@ def test_import(flask_client): assert len(Alias.filter_by(user_id=user.id).all()) == 1 # Onboarding alias # Create domains - domain1 = CustomDomain.create( - user_id=user.id, domain="my-domain.com", verified=True - ) - domain2 = CustomDomain.create( + CustomDomain.create(user_id=user.id, domain="my-domain.com", verified=True) + CustomDomain.create( user_id=user.id, domain="my-destination-domain.com", verified=True ) db.session.commit()