Simpler csv export (#1383)

* Export alias in csv

* reformating

* template

* Improved contributing script and doc

* Updated test

* removed csv export from GDPR export archive

* added test for new route

* fix trailing space

* moved test to new utils file

CONTRIBUTING.md

@@ -62,6 +62,8 @@ To install it in your development environment.
 
 ## Run tests
 
+For most tests, you will need to have ``redis`` installed and started on your machine (listening on port 6379).
+
 ```bash
 sh scripts/run-test.sh
 ```
@@ -80,6 +82,12 @@ To run the code locally, please create a local setting file based on `example.en
 cp example.env .env
 ```
 
+You need to edit your .env to reflect the postgres exposed port, edit the `DB_URI` to:
+
+```
+DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin
+```
+
 Run the postgres database:
 
 ```bash
@@ -198,4 +206,11 @@ python email_handler.py
 swaks --to e1@sl.local --from hey@google.com --server 127.0.0.1:20381
 ```
 
-Now open http://localhost:1080/ (or http://localhost:1080/ for MailHog), you should see the forwarded email.
+Now open http://localhost:1080/ (or http://localhost:1080/ for MailHog), you should see the forwarded email.
+
+## Job runner
+
+Some features require a job handler (such as GDPR data export). To test such feature you need to run the job_runner
+```bash
+python job_runner.py
+```

app/alias_utils.py

@@ -1,8 +1,11 @@
+import csv
+from io import StringIO
 import re
 from typing import Optional, Tuple
 
 from email_validator import validate_email, EmailNotValidError
 from sqlalchemy.exc import IntegrityError, DataError
+from flask import make_response
 
 from app.config import (
     BOUNCE_PREFIX_FOR_REPLY_PHASE,
@@ -364,3 +367,33 @@ def check_alias_prefix(alias_prefix) -> bool:
         return False
 
     return True
+
+
+def alias_export_csv(user, csv_direct_export=False):
+    """
+    Get user aliases as importable CSV file
+    Output:
+        Importable CSV file
+
+    """
+    data = [["alias", "note", "enabled", "mailboxes"]]
+    for alias in Alias.filter_by(user_id=user.id).all():  # type: Alias
+        # Always put the main mailbox first
+        # It is seen a primary while importing
+        alias_mailboxes = alias.mailboxes
+        alias_mailboxes.insert(
+            0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox))
+        )
+
+        mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes])
+        data.append([alias.email, alias.note, alias.enabled, mailboxes])
+
+    si = StringIO()
+    cw = csv.writer(si)
+    cw.writerows(data)
+    if csv_direct_export:
+        return si.getvalue()
+    output = make_response(si.getvalue())
+    output.headers["Content-Disposition"] = "attachment; filename=aliases.csv"
+    output.headers["Content-type"] = "text/csv"
+    return output

app/api/views/export.py

@@ -1,12 +1,9 @@
-import csv
-from io import StringIO
-
 from flask import g
 from flask import jsonify
-from flask import make_response
 
 from app.api.base import api_bp, require_api_auth
 from app.models import Alias, Client, CustomDomain
+from app.alias_utils import alias_export_csv
 
 
 @api_bp.route("/export/data", methods=["GET"])
@@ -49,24 +46,4 @@ def export_aliases():
         Importable CSV file
 
     """
-    user = g.user
-
-    data = [["alias", "note", "enabled", "mailboxes"]]
-    for alias in Alias.filter_by(user_id=user.id).all():  # type: Alias
-        # Always put the main mailbox first
-        # It is seen a primary while importing
-        alias_mailboxes = alias.mailboxes
-        alias_mailboxes.insert(
-            0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox))
-        )
-
-        mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes])
-        data.append([alias.email, alias.note, alias.enabled, mailboxes])
-
-    si = StringIO()
-    cw = csv.writer(si)
-    cw.writerows(data)
-    output = make_response(si.getvalue())
-    output.headers["Content-Disposition"] = "attachment; filename=aliases.csv"
-    output.headers["Content-type"] = "text/csv"
-    return output
+    return alias_export_csv(g.user)

app/dashboard/__init__.py

@@ -6,6 +6,7 @@ from .views import (
     subdomain,
     billing,
     alias_log,
+    alias_export,
     unsubscribe,
     api_key,
     custom_domain,

app/dashboard/views/alias_export.py

@@ -0,0 +1,9 @@
+from app.dashboard.base import dashboard_bp
+from flask_login import login_required, current_user
+from app.alias_utils import alias_export_csv
+
+
+@dashboard_bp.route("/alias_export", methods=["GET"])
+@login_required
+def alias_export_route():
+    return alias_export_csv(current_user)

scripts/reset_local_db.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-export DB_URI=postgresql://myuser:mypassword@localhost:15432/simplelogin
+export DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin
 echo 'drop schema public cascade; create schema public;' | psql  $DB_URI
 
 poetry run alembic upgrade head

scripts/reset_test_db.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-export DB_URI=postgresql://myuser:mypassword@localhost:15432/test
+export DB_URI=postgresql://myuser:mypassword@localhost:35432/test
 echo 'drop schema public cascade; create schema public;' | psql  $DB_URI
 
 poetry run alembic upgrade head

templates/dashboard/setting.html

@@ -702,15 +702,20 @@
         <div class="card">
           <div class="card-body">
             <div class="card-title">
-              Alias import
+              Alias import/export
             </div>
             <div class="mb-3">
               You can import your aliases created on other platforms into SimpleLogin.
+              You can also export your aliases to a readable csv format for a future batch import.
             </div>
             <a href="{{ url_for('dashboard.batch_import_route') }}"
                class="btn btn-outline-primary">
               Batch Import
             </a>
+            <a href="{{ url_for('dashboard.alias_export_route') }}"
+               class="btn btn-outline-secondary">
+              Export Aliases
+            </a>
           </div>
         </div>
         <div class="card">

tests/api/test_import_export.py

@@ -1,105 +1,18 @@
-import csv
-from io import StringIO
-
-from flask import url_for
-
-from app import alias_utils
 from app.db import Session
 from app.import_utils import import_from_csv
 from app.models import (
     CustomDomain,
     Mailbox,
     Alias,
-    AliasMailbox,
     BatchImport,
     File,
 )
-from tests.utils import login, create_new_user, random_domain, random_token
+from tests.utils_test_alias import alias_export
+from tests.utils import login, random_domain, random_token
 
 
 def test_export(flask_client):
-    # Create users
-    user1 = login(flask_client)
-    user2 = create_new_user()
-    Session.commit()
-
-    # Remove onboarding aliases
-    for alias in Alias.filter_by(user_id=user1.id).all():
-        alias_utils.delete_alias(alias, user1)
-    for alias in Alias.filter_by(user_id=user2.id).all():
-        alias_utils.delete_alias(alias, user2)
-    Session.commit()
-
-    # Create domains
-    ok_domain = CustomDomain.create(
-        user_id=user1.id, domain=random_domain(), verified=True
-    )
-    bad_domain = CustomDomain.create(
-        user_id=user2.id, domain=random_domain(), verified=True
-    )
-    Session.commit()
-
-    # Create mailboxes
-    mailbox1 = Mailbox.create(
-        user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
-    )
-    mailbox2 = Mailbox.create(
-        user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
-    )
-    badmailbox1 = Mailbox.create(
-        user_id=user2.id,
-        email=f"{random_token()}@{bad_domain.domain}",
-        verified=True,
-    )
-    Session.commit()
-
-    # Create aliases
-    alias1 = Alias.create(
-        user_id=user1.id,
-        email=f"{random_token()}@my-domain.com",
-        note="Used on eBay",
-        mailbox_id=mailbox1.id,
-    )
-    alias2 = Alias.create(
-        user_id=user1.id,
-        email=f"{random_token()}@my-domain.com",
-        note="Used on Facebook, Instagram.",
-        mailbox_id=mailbox1.id,
-    )
-    Alias.create(
-        user_id=user2.id,
-        email=f"{random_token()}@my-domain.com",
-        note="Should not appear",
-        mailbox_id=badmailbox1.id,
-    )
-    Session.commit()
-
-    # Add second mailbox to an alias
-    AliasMailbox.create(
-        alias_id=alias2.id,
-        mailbox_id=mailbox2.id,
-    )
-    Session.commit()
-
-    # Export
-    r = flask_client.get(url_for("api.export_aliases"))
-    assert r.status_code == 200
-    assert r.mimetype == "text/csv"
-    csv_data = csv.DictReader(StringIO(r.data.decode("utf-8")))
-    found_aliases = set()
-    for row in csv_data:
-        found_aliases.add(row["alias"])
-        if row["alias"] == alias1.email:
-            assert alias1.note == row["note"]
-            assert "True" == row["enabled"]
-            assert mailbox1.email == row["mailboxes"]
-        elif row["alias"] == alias2.email:
-            assert alias2.note == row["note"]
-            assert "True" == row["enabled"]
-            assert f"{mailbox1.email} {mailbox2.email}" == row["mailboxes"]
-        else:
-            raise AssertionError("Unknown alias")
-    assert set((alias1.email, alias2.email)) == found_aliases
+    alias_export(flask_client, "api.export_aliases")
 
 
 def test_import_no_mailboxes_no_domains(flask_client):

tests/dashboard/test_alias_csv_export.py

@@ -0,0 +1,5 @@
+from tests.utils_test_alias import alias_export
+
+
+def test_alias_export(flask_client):
+    alias_export(flask_client, "dashboard.alias_export_route")

tests/utils_test_alias.py

@@ -0,0 +1,95 @@
+import csv
+from io import StringIO
+
+from flask import url_for
+
+from app.alias_utils import delete_alias
+from app.db import Session
+from app.models import Alias, CustomDomain, Mailbox, AliasMailbox
+
+from tests.utils import login, create_new_user, random_domain, random_token
+
+
+def alias_export(flask_client, target_url):
+    # Create users
+    user1 = login(flask_client)
+    user2 = create_new_user()
+    Session.commit()
+
+    # Remove onboarding aliases
+    for alias in Alias.filter_by(user_id=user1.id).all():
+        delete_alias(alias, user1)
+    for alias in Alias.filter_by(user_id=user2.id).all():
+        delete_alias(alias, user2)
+    Session.commit()
+
+    # Create domains
+    ok_domain = CustomDomain.create(
+        user_id=user1.id, domain=random_domain(), verified=True
+    )
+    bad_domain = CustomDomain.create(
+        user_id=user2.id, domain=random_domain(), verified=True
+    )
+    Session.commit()
+
+    # Create mailboxes
+    mailbox1 = Mailbox.create(
+        user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
+    )
+    mailbox2 = Mailbox.create(
+        user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
+    )
+    badmailbox1 = Mailbox.create(
+        user_id=user2.id,
+        email=f"{random_token()}@{bad_domain.domain}",
+        verified=True,
+    )
+    Session.commit()
+
+    # Create aliases
+    alias1 = Alias.create(
+        user_id=user1.id,
+        email=f"{random_token()}@my-domain.com",
+        note="Used on eBay",
+        mailbox_id=mailbox1.id,
+    )
+    alias2 = Alias.create(
+        user_id=user1.id,
+        email=f"{random_token()}@my-domain.com",
+        note="Used on Facebook, Instagram.",
+        mailbox_id=mailbox1.id,
+    )
+    Alias.create(
+        user_id=user2.id,
+        email=f"{random_token()}@my-domain.com",
+        note="Should not appear",
+        mailbox_id=badmailbox1.id,
+    )
+    Session.commit()
+
+    # Add second mailbox to an alias
+    AliasMailbox.create(
+        alias_id=alias2.id,
+        mailbox_id=mailbox2.id,
+    )
+    Session.commit()
+
+    # Export
+    r = flask_client.get(url_for(target_url))
+    assert r.status_code == 200
+    assert r.mimetype == "text/csv"
+    csv_data = csv.DictReader(StringIO(r.data.decode("utf-8")))
+    found_aliases = set()
+    for row in csv_data:
+        found_aliases.add(row["alias"])
+        if row["alias"] == alias1.email:
+            assert alias1.note == row["note"]
+            assert "True" == row["enabled"]
+            assert mailbox1.email == row["mailboxes"]
+        elif row["alias"] == alias2.email:
+            assert alias2.note == row["note"]
+            assert "True" == row["enabled"]
+            assert f"{mailbox1.email} {mailbox2.email}" == row["mailboxes"]
+        else:
+            raise AssertionError("Unknown alias")
+    assert set((alias1.email, alias2.email)) == found_aliases