mirror of
https://github.com/simple-login/app.git
synced 2024-09-27 20:31:30 +02:00
Simpler csv export (#1383)
* Export alias in csv * reformating * template * Improved contributing script and doc * Updated test * removed csv export from GDPR export archive * added test for new route * fix trailing space * moved test to new utils file
This commit is contained in:
parent
0fbe576c44
commit
b849d1cfa7
@ -62,6 +62,8 @@ To install it in your development environment.
|
|||||||
|
|
||||||
## Run tests
|
## Run tests
|
||||||
|
|
||||||
|
For most tests, you will need to have ``redis`` installed and started on your machine (listening on port 6379).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sh scripts/run-test.sh
|
sh scripts/run-test.sh
|
||||||
```
|
```
|
||||||
@ -80,6 +82,12 @@ To run the code locally, please create a local setting file based on `example.en
|
|||||||
cp example.env .env
|
cp example.env .env
|
||||||
```
|
```
|
||||||
|
|
||||||
|
You need to edit your .env to reflect the postgres exposed port, edit the `DB_URI` to:
|
||||||
|
|
||||||
|
```
|
||||||
|
DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin
|
||||||
|
```
|
||||||
|
|
||||||
Run the postgres database:
|
Run the postgres database:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@ -199,3 +207,10 @@ swaks --to e1@sl.local --from hey@google.com --server 127.0.0.1:20381
|
|||||||
```
|
```
|
||||||
|
|
||||||
Now open http://localhost:1080/ (or http://localhost:1080/ for MailHog), you should see the forwarded email.
|
Now open http://localhost:1080/ (or http://localhost:1080/ for MailHog), you should see the forwarded email.
|
||||||
|
|
||||||
|
## Job runner
|
||||||
|
|
||||||
|
Some features require a job handler (such as GDPR data export). To test such feature you need to run the job_runner
|
||||||
|
```bash
|
||||||
|
python job_runner.py
|
||||||
|
```
|
@ -1,8 +1,11 @@
|
|||||||
|
import csv
|
||||||
|
from io import StringIO
|
||||||
import re
|
import re
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
|
|
||||||
from email_validator import validate_email, EmailNotValidError
|
from email_validator import validate_email, EmailNotValidError
|
||||||
from sqlalchemy.exc import IntegrityError, DataError
|
from sqlalchemy.exc import IntegrityError, DataError
|
||||||
|
from flask import make_response
|
||||||
|
|
||||||
from app.config import (
|
from app.config import (
|
||||||
BOUNCE_PREFIX_FOR_REPLY_PHASE,
|
BOUNCE_PREFIX_FOR_REPLY_PHASE,
|
||||||
@ -364,3 +367,33 @@ def check_alias_prefix(alias_prefix) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def alias_export_csv(user, csv_direct_export=False):
|
||||||
|
"""
|
||||||
|
Get user aliases as importable CSV file
|
||||||
|
Output:
|
||||||
|
Importable CSV file
|
||||||
|
|
||||||
|
"""
|
||||||
|
data = [["alias", "note", "enabled", "mailboxes"]]
|
||||||
|
for alias in Alias.filter_by(user_id=user.id).all(): # type: Alias
|
||||||
|
# Always put the main mailbox first
|
||||||
|
# It is seen a primary while importing
|
||||||
|
alias_mailboxes = alias.mailboxes
|
||||||
|
alias_mailboxes.insert(
|
||||||
|
0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox))
|
||||||
|
)
|
||||||
|
|
||||||
|
mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes])
|
||||||
|
data.append([alias.email, alias.note, alias.enabled, mailboxes])
|
||||||
|
|
||||||
|
si = StringIO()
|
||||||
|
cw = csv.writer(si)
|
||||||
|
cw.writerows(data)
|
||||||
|
if csv_direct_export:
|
||||||
|
return si.getvalue()
|
||||||
|
output = make_response(si.getvalue())
|
||||||
|
output.headers["Content-Disposition"] = "attachment; filename=aliases.csv"
|
||||||
|
output.headers["Content-type"] = "text/csv"
|
||||||
|
return output
|
||||||
|
@ -1,12 +1,9 @@
|
|||||||
import csv
|
|
||||||
from io import StringIO
|
|
||||||
|
|
||||||
from flask import g
|
from flask import g
|
||||||
from flask import jsonify
|
from flask import jsonify
|
||||||
from flask import make_response
|
|
||||||
|
|
||||||
from app.api.base import api_bp, require_api_auth
|
from app.api.base import api_bp, require_api_auth
|
||||||
from app.models import Alias, Client, CustomDomain
|
from app.models import Alias, Client, CustomDomain
|
||||||
|
from app.alias_utils import alias_export_csv
|
||||||
|
|
||||||
|
|
||||||
@api_bp.route("/export/data", methods=["GET"])
|
@api_bp.route("/export/data", methods=["GET"])
|
||||||
@ -49,24 +46,4 @@ def export_aliases():
|
|||||||
Importable CSV file
|
Importable CSV file
|
||||||
|
|
||||||
"""
|
"""
|
||||||
user = g.user
|
return alias_export_csv(g.user)
|
||||||
|
|
||||||
data = [["alias", "note", "enabled", "mailboxes"]]
|
|
||||||
for alias in Alias.filter_by(user_id=user.id).all(): # type: Alias
|
|
||||||
# Always put the main mailbox first
|
|
||||||
# It is seen a primary while importing
|
|
||||||
alias_mailboxes = alias.mailboxes
|
|
||||||
alias_mailboxes.insert(
|
|
||||||
0, alias_mailboxes.pop(alias_mailboxes.index(alias.mailbox))
|
|
||||||
)
|
|
||||||
|
|
||||||
mailboxes = " ".join([mailbox.email for mailbox in alias_mailboxes])
|
|
||||||
data.append([alias.email, alias.note, alias.enabled, mailboxes])
|
|
||||||
|
|
||||||
si = StringIO()
|
|
||||||
cw = csv.writer(si)
|
|
||||||
cw.writerows(data)
|
|
||||||
output = make_response(si.getvalue())
|
|
||||||
output.headers["Content-Disposition"] = "attachment; filename=aliases.csv"
|
|
||||||
output.headers["Content-type"] = "text/csv"
|
|
||||||
return output
|
|
||||||
|
@ -6,6 +6,7 @@ from .views import (
|
|||||||
subdomain,
|
subdomain,
|
||||||
billing,
|
billing,
|
||||||
alias_log,
|
alias_log,
|
||||||
|
alias_export,
|
||||||
unsubscribe,
|
unsubscribe,
|
||||||
api_key,
|
api_key,
|
||||||
custom_domain,
|
custom_domain,
|
||||||
|
9
app/dashboard/views/alias_export.py
Normal file
9
app/dashboard/views/alias_export.py
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
from app.dashboard.base import dashboard_bp
|
||||||
|
from flask_login import login_required, current_user
|
||||||
|
from app.alias_utils import alias_export_csv
|
||||||
|
|
||||||
|
|
||||||
|
@dashboard_bp.route("/alias_export", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def alias_export_route():
|
||||||
|
return alias_export_csv(current_user)
|
@ -1,6 +1,6 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
|
|
||||||
export DB_URI=postgresql://myuser:mypassword@localhost:15432/simplelogin
|
export DB_URI=postgresql://myuser:mypassword@localhost:35432/simplelogin
|
||||||
echo 'drop schema public cascade; create schema public;' | psql $DB_URI
|
echo 'drop schema public cascade; create schema public;' | psql $DB_URI
|
||||||
|
|
||||||
poetry run alembic upgrade head
|
poetry run alembic upgrade head
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
|
|
||||||
export DB_URI=postgresql://myuser:mypassword@localhost:15432/test
|
export DB_URI=postgresql://myuser:mypassword@localhost:35432/test
|
||||||
echo 'drop schema public cascade; create schema public;' | psql $DB_URI
|
echo 'drop schema public cascade; create schema public;' | psql $DB_URI
|
||||||
|
|
||||||
poetry run alembic upgrade head
|
poetry run alembic upgrade head
|
||||||
|
@ -702,15 +702,20 @@
|
|||||||
<div class="card">
|
<div class="card">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<div class="card-title">
|
<div class="card-title">
|
||||||
Alias import
|
Alias import/export
|
||||||
</div>
|
</div>
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
You can import your aliases created on other platforms into SimpleLogin.
|
You can import your aliases created on other platforms into SimpleLogin.
|
||||||
|
You can also export your aliases to a readable csv format for a future batch import.
|
||||||
</div>
|
</div>
|
||||||
<a href="{{ url_for('dashboard.batch_import_route') }}"
|
<a href="{{ url_for('dashboard.batch_import_route') }}"
|
||||||
class="btn btn-outline-primary">
|
class="btn btn-outline-primary">
|
||||||
Batch Import
|
Batch Import
|
||||||
</a>
|
</a>
|
||||||
|
<a href="{{ url_for('dashboard.alias_export_route') }}"
|
||||||
|
class="btn btn-outline-secondary">
|
||||||
|
Export Aliases
|
||||||
|
</a>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="card">
|
<div class="card">
|
||||||
|
@ -1,105 +1,18 @@
|
|||||||
import csv
|
|
||||||
from io import StringIO
|
|
||||||
|
|
||||||
from flask import url_for
|
|
||||||
|
|
||||||
from app import alias_utils
|
|
||||||
from app.db import Session
|
from app.db import Session
|
||||||
from app.import_utils import import_from_csv
|
from app.import_utils import import_from_csv
|
||||||
from app.models import (
|
from app.models import (
|
||||||
CustomDomain,
|
CustomDomain,
|
||||||
Mailbox,
|
Mailbox,
|
||||||
Alias,
|
Alias,
|
||||||
AliasMailbox,
|
|
||||||
BatchImport,
|
BatchImport,
|
||||||
File,
|
File,
|
||||||
)
|
)
|
||||||
from tests.utils import login, create_new_user, random_domain, random_token
|
from tests.utils_test_alias import alias_export
|
||||||
|
from tests.utils import login, random_domain, random_token
|
||||||
|
|
||||||
|
|
||||||
def test_export(flask_client):
|
def test_export(flask_client):
|
||||||
# Create users
|
alias_export(flask_client, "api.export_aliases")
|
||||||
user1 = login(flask_client)
|
|
||||||
user2 = create_new_user()
|
|
||||||
Session.commit()
|
|
||||||
|
|
||||||
# Remove onboarding aliases
|
|
||||||
for alias in Alias.filter_by(user_id=user1.id).all():
|
|
||||||
alias_utils.delete_alias(alias, user1)
|
|
||||||
for alias in Alias.filter_by(user_id=user2.id).all():
|
|
||||||
alias_utils.delete_alias(alias, user2)
|
|
||||||
Session.commit()
|
|
||||||
|
|
||||||
# Create domains
|
|
||||||
ok_domain = CustomDomain.create(
|
|
||||||
user_id=user1.id, domain=random_domain(), verified=True
|
|
||||||
)
|
|
||||||
bad_domain = CustomDomain.create(
|
|
||||||
user_id=user2.id, domain=random_domain(), verified=True
|
|
||||||
)
|
|
||||||
Session.commit()
|
|
||||||
|
|
||||||
# Create mailboxes
|
|
||||||
mailbox1 = Mailbox.create(
|
|
||||||
user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
|
|
||||||
)
|
|
||||||
mailbox2 = Mailbox.create(
|
|
||||||
user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
|
|
||||||
)
|
|
||||||
badmailbox1 = Mailbox.create(
|
|
||||||
user_id=user2.id,
|
|
||||||
email=f"{random_token()}@{bad_domain.domain}",
|
|
||||||
verified=True,
|
|
||||||
)
|
|
||||||
Session.commit()
|
|
||||||
|
|
||||||
# Create aliases
|
|
||||||
alias1 = Alias.create(
|
|
||||||
user_id=user1.id,
|
|
||||||
email=f"{random_token()}@my-domain.com",
|
|
||||||
note="Used on eBay",
|
|
||||||
mailbox_id=mailbox1.id,
|
|
||||||
)
|
|
||||||
alias2 = Alias.create(
|
|
||||||
user_id=user1.id,
|
|
||||||
email=f"{random_token()}@my-domain.com",
|
|
||||||
note="Used on Facebook, Instagram.",
|
|
||||||
mailbox_id=mailbox1.id,
|
|
||||||
)
|
|
||||||
Alias.create(
|
|
||||||
user_id=user2.id,
|
|
||||||
email=f"{random_token()}@my-domain.com",
|
|
||||||
note="Should not appear",
|
|
||||||
mailbox_id=badmailbox1.id,
|
|
||||||
)
|
|
||||||
Session.commit()
|
|
||||||
|
|
||||||
# Add second mailbox to an alias
|
|
||||||
AliasMailbox.create(
|
|
||||||
alias_id=alias2.id,
|
|
||||||
mailbox_id=mailbox2.id,
|
|
||||||
)
|
|
||||||
Session.commit()
|
|
||||||
|
|
||||||
# Export
|
|
||||||
r = flask_client.get(url_for("api.export_aliases"))
|
|
||||||
assert r.status_code == 200
|
|
||||||
assert r.mimetype == "text/csv"
|
|
||||||
csv_data = csv.DictReader(StringIO(r.data.decode("utf-8")))
|
|
||||||
found_aliases = set()
|
|
||||||
for row in csv_data:
|
|
||||||
found_aliases.add(row["alias"])
|
|
||||||
if row["alias"] == alias1.email:
|
|
||||||
assert alias1.note == row["note"]
|
|
||||||
assert "True" == row["enabled"]
|
|
||||||
assert mailbox1.email == row["mailboxes"]
|
|
||||||
elif row["alias"] == alias2.email:
|
|
||||||
assert alias2.note == row["note"]
|
|
||||||
assert "True" == row["enabled"]
|
|
||||||
assert f"{mailbox1.email} {mailbox2.email}" == row["mailboxes"]
|
|
||||||
else:
|
|
||||||
raise AssertionError("Unknown alias")
|
|
||||||
assert set((alias1.email, alias2.email)) == found_aliases
|
|
||||||
|
|
||||||
|
|
||||||
def test_import_no_mailboxes_no_domains(flask_client):
|
def test_import_no_mailboxes_no_domains(flask_client):
|
||||||
|
5
tests/dashboard/test_alias_csv_export.py
Normal file
5
tests/dashboard/test_alias_csv_export.py
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
from tests.utils_test_alias import alias_export
|
||||||
|
|
||||||
|
|
||||||
|
def test_alias_export(flask_client):
|
||||||
|
alias_export(flask_client, "dashboard.alias_export_route")
|
95
tests/utils_test_alias.py
Normal file
95
tests/utils_test_alias.py
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
import csv
|
||||||
|
from io import StringIO
|
||||||
|
|
||||||
|
from flask import url_for
|
||||||
|
|
||||||
|
from app.alias_utils import delete_alias
|
||||||
|
from app.db import Session
|
||||||
|
from app.models import Alias, CustomDomain, Mailbox, AliasMailbox
|
||||||
|
|
||||||
|
from tests.utils import login, create_new_user, random_domain, random_token
|
||||||
|
|
||||||
|
|
||||||
|
def alias_export(flask_client, target_url):
|
||||||
|
# Create users
|
||||||
|
user1 = login(flask_client)
|
||||||
|
user2 = create_new_user()
|
||||||
|
Session.commit()
|
||||||
|
|
||||||
|
# Remove onboarding aliases
|
||||||
|
for alias in Alias.filter_by(user_id=user1.id).all():
|
||||||
|
delete_alias(alias, user1)
|
||||||
|
for alias in Alias.filter_by(user_id=user2.id).all():
|
||||||
|
delete_alias(alias, user2)
|
||||||
|
Session.commit()
|
||||||
|
|
||||||
|
# Create domains
|
||||||
|
ok_domain = CustomDomain.create(
|
||||||
|
user_id=user1.id, domain=random_domain(), verified=True
|
||||||
|
)
|
||||||
|
bad_domain = CustomDomain.create(
|
||||||
|
user_id=user2.id, domain=random_domain(), verified=True
|
||||||
|
)
|
||||||
|
Session.commit()
|
||||||
|
|
||||||
|
# Create mailboxes
|
||||||
|
mailbox1 = Mailbox.create(
|
||||||
|
user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
|
||||||
|
)
|
||||||
|
mailbox2 = Mailbox.create(
|
||||||
|
user_id=user1.id, email=f"{random_token()}@{ok_domain.domain}", verified=True
|
||||||
|
)
|
||||||
|
badmailbox1 = Mailbox.create(
|
||||||
|
user_id=user2.id,
|
||||||
|
email=f"{random_token()}@{bad_domain.domain}",
|
||||||
|
verified=True,
|
||||||
|
)
|
||||||
|
Session.commit()
|
||||||
|
|
||||||
|
# Create aliases
|
||||||
|
alias1 = Alias.create(
|
||||||
|
user_id=user1.id,
|
||||||
|
email=f"{random_token()}@my-domain.com",
|
||||||
|
note="Used on eBay",
|
||||||
|
mailbox_id=mailbox1.id,
|
||||||
|
)
|
||||||
|
alias2 = Alias.create(
|
||||||
|
user_id=user1.id,
|
||||||
|
email=f"{random_token()}@my-domain.com",
|
||||||
|
note="Used on Facebook, Instagram.",
|
||||||
|
mailbox_id=mailbox1.id,
|
||||||
|
)
|
||||||
|
Alias.create(
|
||||||
|
user_id=user2.id,
|
||||||
|
email=f"{random_token()}@my-domain.com",
|
||||||
|
note="Should not appear",
|
||||||
|
mailbox_id=badmailbox1.id,
|
||||||
|
)
|
||||||
|
Session.commit()
|
||||||
|
|
||||||
|
# Add second mailbox to an alias
|
||||||
|
AliasMailbox.create(
|
||||||
|
alias_id=alias2.id,
|
||||||
|
mailbox_id=mailbox2.id,
|
||||||
|
)
|
||||||
|
Session.commit()
|
||||||
|
|
||||||
|
# Export
|
||||||
|
r = flask_client.get(url_for(target_url))
|
||||||
|
assert r.status_code == 200
|
||||||
|
assert r.mimetype == "text/csv"
|
||||||
|
csv_data = csv.DictReader(StringIO(r.data.decode("utf-8")))
|
||||||
|
found_aliases = set()
|
||||||
|
for row in csv_data:
|
||||||
|
found_aliases.add(row["alias"])
|
||||||
|
if row["alias"] == alias1.email:
|
||||||
|
assert alias1.note == row["note"]
|
||||||
|
assert "True" == row["enabled"]
|
||||||
|
assert mailbox1.email == row["mailboxes"]
|
||||||
|
elif row["alias"] == alias2.email:
|
||||||
|
assert alias2.note == row["note"]
|
||||||
|
assert "True" == row["enabled"]
|
||||||
|
assert f"{mailbox1.email} {mailbox2.email}" == row["mailboxes"]
|
||||||
|
else:
|
||||||
|
raise AssertionError("Unknown alias")
|
||||||
|
assert set((alias1.email, alias2.email)) == found_aliases
|
Loading…
Reference in New Issue
Block a user