From b849d1cfa79a2ecbdcc6d83ae940b7562d28f5a5 Mon Sep 17 00:00:00 2001 From: Spitfireap <45575529+Spitfireap@users.noreply.github.com> Date: Wed, 23 Nov 2022 13:51:08 +0100 Subject: [PATCH] Simpler csv export (#1383) * Export alias in csv * reformating * template * Improved contributing script and doc * Updated test * removed csv export from GDPR export archive * added test for new route * fix trailing space * moved test to new utils file --- CONTRIBUTING.md | 17 ++++- app/alias_utils.py | 33 ++++++++ app/api/views/export.py | 27 +------ app/dashboard/__init__.py | 1 + app/dashboard/views/alias_export.py | 9 +++ scripts/reset_local_db.sh | 2 +- scripts/reset_test_db.sh | 2 +- templates/dashboard/setting.html | 7 +- tests/api/test_import_export.py | 93 +---------------------- tests/dashboard/test_alias_csv_export.py | 5 ++ tests/utils_test_alias.py | 95 ++++++++++++++++++++++++ 11 files changed, 172 insertions(+), 119 deletions(-) create mode 100644 app/dashboard/views/alias_export.py create mode 100644 tests/dashboard/test_alias_csv_export.py create mode 100644 tests/utils_test_alias.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77d8ae80..c622af42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,8 @@ To install it in your development environment. ## Run tests +For most tests, you will need to have ``redis`` installed and started on your machine (listening on port 6379). + ```bash sh scripts/run-test.sh ``` @@ -80,6 +82,12 @@ To run the code locally, please create a local setting file based on `example.en cp example.env .env ``` +You need to edit your .env to reflect the postgres exposed port, edit the `DB_URI` to: + +``` +DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin +``` + Run the postgres database: ```bash @@ -198,4 +206,11 @@ python email_handler.py swaks --to e1@sl.local --from hey@google.com --server 127.0.0.1:20381 ``` -Now open http://localhost:1080/ (or http://localhost:1080/ for MailHog), you should see the forwarded email. \ No newline at end of file +Now open http://localhost:1080/ (or http://localhost:1080/ for MailHog), you should see the forwarded email. + +## Job runner + +Some features require a job handler (such as GDPR data export). To test such feature you need to run the job_runner +```bash +python job_runner.py +``` \ No newline at end of file diff --git a/app/alias_utils.py b/app/alias_utils.py index c431e377..7b1305bb 100644 --- a/app/alias_utils.py +++ b/app/alias_utils.py @@ -1,8 +1,11 @@ +import csv +from io import StringIO import re from typing import Optional, Tuple from email_validator import validate_email, EmailNotValidError from sqlalchemy.exc import IntegrityError, DataError +from flask import make_response from app.config import ( BOUNCE_PREFIX_FOR_REPLY_PHASE, @@ -364,3 +367,33 @@ def check_alias_prefix(alias_prefix) -> bool: return False return True + + +def alias_export_csv(user, csv_direct_export=False): + """ + Get user aliases as importable CSV file + Output: + Importable CSV file + + """ + data = [["alias", "note", "enabled", "mailboxes"]] + for alias in Alias.filter_by(user_id=user.id).all(): # type: Alias + # Always put the main mailbox first + # It is seen a primary while importing + alias_mailboxes = alias.mailboxes + alias_mailboxes.insert( + 0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox)) + ) + + mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes]) + data.append([alias.email, alias.note, alias.enabled, mailboxes]) + + si = StringIO() + cw = csv.writer(si) + cw.writerows(data) + if csv_direct_export: + return si.getvalue() + output = make_response(si.getvalue()) + output.headers["Content-Disposition"] = "attachment; filename=aliases.csv" + output.headers["Content-type"] = "text/csv" + return output diff --git a/app/api/views/export.py b/app/api/views/export.py index e8bace4f..8f5b3347 100644 --- a/app/api/views/export.py +++ b/app/api/views/export.py @@ -1,12 +1,9 @@ -import csv -from io import StringIO - from flask import g from flask import jsonify -from flask import make_response from app.api.base import api_bp, require_api_auth from app.models import Alias, Client, CustomDomain +from app.alias_utils import alias_export_csv @api_bp.route("/export/data", methods=["GET"]) @@ -49,24 +46,4 @@ def export_aliases(): Importable CSV file """ - user = g.user - - data = [["alias", "note", "enabled", "mailboxes"]] - for alias in Alias.filter_by(user_id=user.id).all(): # type: Alias - # Always put the main mailbox first - # It is seen a primary while importing - alias_mailboxes = alias.mailboxes - alias_mailboxes.insert( - 0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox)) - ) - - mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes]) - data.append([alias.email, alias.note, alias.enabled, mailboxes]) - - si = StringIO() - cw = csv.writer(si) - cw.writerows(data) - output = make_response(si.getvalue()) - output.headers["Content-Disposition"] = "attachment; filename=aliases.csv" - output.headers["Content-type"] = "text/csv" - return output + return alias_export_csv(g.user) diff --git a/app/dashboard/__init__.py b/app/dashboard/__init__.py index cc19eaa2..ebfc38d6 100644 --- a/app/dashboard/__init__.py +++ b/app/dashboard/__init__.py @@ -6,6 +6,7 @@ from .views import ( subdomain, billing, alias_log, + alias_export, unsubscribe, api_key, custom_domain, diff --git a/app/dashboard/views/alias_export.py b/app/dashboard/views/alias_export.py new file mode 100644 index 00000000..9d48b382 --- /dev/null +++ b/app/dashboard/views/alias_export.py @@ -0,0 +1,9 @@ +from app.dashboard.base import dashboard_bp +from flask_login import login_required, current_user +from app.alias_utils import alias_export_csv + + +@dashboard_bp.route("/alias_export", methods=["GET"]) +@login_required +def alias_export_route(): + return alias_export_csv(current_user) diff --git a/scripts/reset_local_db.sh b/scripts/reset_local_db.sh index 422c2a8b..cf8e4f0f 100755 --- a/scripts/reset_local_db.sh +++ b/scripts/reset_local_db.sh @@ -1,6 +1,6 @@ #!/bin/sh -export DB_URI=postgresql://myuser:mypassword@localhost:15432/simplelogin +export DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin echo 'drop schema public cascade; create schema public;' | psql $DB_URI poetry run alembic upgrade head diff --git a/scripts/reset_test_db.sh b/scripts/reset_test_db.sh index 25466010..a24edfe2 100755 --- a/scripts/reset_test_db.sh +++ b/scripts/reset_test_db.sh @@ -1,6 +1,6 @@ #!/bin/sh -export DB_URI=postgresql://myuser:mypassword@localhost:15432/test +export DB_URI=postgresql://myuser:mypassword@localhost:35432/test echo 'drop schema public cascade; create schema public;' | psql $DB_URI poetry run alembic upgrade head diff --git a/templates/dashboard/setting.html b/templates/dashboard/setting.html index df257911..43550576 100644 --- a/templates/dashboard/setting.html +++ b/templates/dashboard/setting.html @@ -702,15 +702,20 @@