Send full user report asynchronously on request (#1029)
* Send full user report asynchronously * Fix test * Filter some fields before exporting * Fix: Domain -> CustomDomain * format settings html * not include RefusedEmail as they are not usable by user and are automatically deleted * send the export to the user email * change email and setting wording * fix user can only export data once * remove alias export section * remove unused import * fix flake8 Co-authored-by: Adrià Casajús <adria.casajus@proton.ch> Co-authored-by: Son <nguyenkims@users.noreply.github.com>
This commit is contained in:
Adrià Casajús
GitHub
parent
cbd44c01f5
commit
e688f04d6b
|
|
@ -263,6 +263,7 @@ JOB_BATCH_IMPORT = "batch-import"
|
|||
JOB_DELETE_ACCOUNT = "delete-account"
|
||||
JOB_DELETE_MAILBOX = "delete-mailbox"
|
||||
JOB_DELETE_DOMAIN = "delete-domain"
|
||||
JOB_SEND_USER_REPORT = "send-user-report"
|
||||
|
||||
# for pagination
|
||||
PAGE_LIMIT = 20
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from app.email_utils import (
|
|||
personal_email_already_used,
|
||||
)
|
||||
from app.errors import ProtonPartnerNotSetUp
|
||||
from app.jobs.export_user_data_job import ExportUserDataJob
|
||||
from app.log import LOG
|
||||
from app.models import (
|
||||
BlockBehaviourEnum,
|
||||
|
|
@ -348,10 +349,14 @@ def setting():
|
|||
Session.commit()
|
||||
flash("Your preference has been updated", "success")
|
||||
return redirect(url_for("dashboard.setting"))
|
||||
elif request.form.get("form-name") == "export-data":
|
||||
return redirect(url_for("api.export_data"))
|
||||
elif request.form.get("form-name") == "export-alias":
|
||||
return redirect(url_for("api.export_aliases"))
|
||||
elif request.form.get("form-name") == "send-full-user-report":
|
||||
if ExportUserDataJob(current_user).store_job_in_db():
|
||||
flash(
|
||||
"You will receive your SimpleLogin data via email shortly",
|
||||
"success",
|
||||
)
|
||||
else:
|
||||
flash("An export of your data is currently in progress", "error")
|
||||
|
||||
manual_sub = ManualSubscription.get_by(user_id=current_user.id)
|
||||
apple_sub = AppleSubscription.get_by(user_id=current_user.id)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,177 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
from email.mime.application import MIMEApplication
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from io import BytesIO
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
import arrow
|
||||
import sqlalchemy
|
||||
|
||||
from app import config
|
||||
from app.db import Session
|
||||
from app.email import headers
|
||||
from app.email_utils import generate_verp_email, render
|
||||
from app.mail_sender import sl_sendmail
|
||||
from app.models import (
|
||||
Alias,
|
||||
Contact,
|
||||
Mailbox,
|
||||
Directory,
|
||||
EmailLog,
|
||||
CustomDomain,
|
||||
RefusedEmail,
|
||||
Base,
|
||||
User,
|
||||
EnumE,
|
||||
TransactionalEmail,
|
||||
VerpType,
|
||||
Job,
|
||||
)
|
||||
|
||||
|
||||
class ExportUserDataJob:
|
||||
|
||||
REMOVE_FIELDS = {
|
||||
"User": ("otp_secret",),
|
||||
"Alias": ("ts_vector", "transfer_token", "hibp_last_check"),
|
||||
"CustomDomain": ("ownership_txt_token",),
|
||||
}
|
||||
|
||||
def __init__(self, user: User):
|
||||
self._user: User = user
|
||||
|
||||
def _get_paginated_model(self, model_class, page_size=50) -> List:
|
||||
objects = []
|
||||
page = 0
|
||||
db_objects = []
|
||||
while page == 0 or len(db_objects) == page_size:
|
||||
db_objects = (
|
||||
Session.query(model_class)
|
||||
.filter(model_class.user_id == self._user.id)
|
||||
.order_by(model_class.id)
|
||||
.limit(page_size)
|
||||
.offset(page * page_size)
|
||||
.all()
|
||||
)
|
||||
objects.extend(db_objects)
|
||||
page += 1
|
||||
return objects
|
||||
|
||||
def _get_aliases(self) -> List[Alias]:
|
||||
return self._get_paginated_model(Alias)
|
||||
|
||||
def _get_mailboxes(self) -> List[Mailbox]:
|
||||
return self._get_paginated_model(Mailbox)
|
||||
|
||||
def _get_contacts(self) -> List[Contact]:
|
||||
return self._get_paginated_model(Contact)
|
||||
|
||||
def _get_directories(self) -> List[Directory]:
|
||||
return self._get_paginated_model(Directory)
|
||||
|
||||
def _get_email_logs(self) -> List[EmailLog]:
|
||||
return self._get_paginated_model(EmailLog)
|
||||
|
||||
def _get_domains(self) -> List[CustomDomain]:
|
||||
return self._get_paginated_model(CustomDomain)
|
||||
|
||||
def _get_refused_emails(self) -> List[RefusedEmail]:
|
||||
return self._get_paginated_model(RefusedEmail)
|
||||
|
||||
@classmethod
|
||||
def _model_to_dict(cls, object: Base) -> Dict:
|
||||
data = {}
|
||||
fields_to_filter = cls.REMOVE_FIELDS.get(object.__class__.__name__, ())
|
||||
for column in object.__table__.columns:
|
||||
if column.name in fields_to_filter:
|
||||
continue
|
||||
value = getattr(object, column.name)
|
||||
if isinstance(value, arrow.Arrow):
|
||||
value = value.isoformat()
|
||||
if issubclass(value.__class__, EnumE):
|
||||
value = value.value
|
||||
data[column.name] = value
|
||||
return data
|
||||
|
||||
def _build_zip(self) -> BytesIO:
|
||||
memfile = BytesIO()
|
||||
with zipfile.ZipFile(memfile, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr(
|
||||
"user.json", json.dumps(ExportUserDataJob._model_to_dict(self._user))
|
||||
)
|
||||
for model_name, get_models in [
|
||||
("aliases", self._get_aliases),
|
||||
("mailboxes", self._get_mailboxes),
|
||||
("contacts", self._get_contacts),
|
||||
("directories", self._get_directories),
|
||||
("domains", self._get_domains),
|
||||
("email_logs", self._get_email_logs),
|
||||
# not include RefusedEmail as they are not usable by user and are automatically deleted
|
||||
# ("refused_emails", self._get_refused_emails),
|
||||
]:
|
||||
model_objs = get_models()
|
||||
data = json.dumps(
|
||||
[
|
||||
ExportUserDataJob._model_to_dict(model_obj)
|
||||
for model_obj in model_objs
|
||||
]
|
||||
)
|
||||
zf.writestr(f"{model_name}.json", data)
|
||||
memfile.seek(0)
|
||||
return memfile
|
||||
|
||||
def run(self):
|
||||
zipped_contents = self._build_zip()
|
||||
|
||||
to_email = self._user.email
|
||||
|
||||
msg = MIMEMultipart()
|
||||
msg[headers.SUBJECT] = "Your SimpleLogin data"
|
||||
msg[headers.FROM] = f'"SimpleLogin (noreply)" <{config.NOREPLY}>'
|
||||
msg[headers.TO] = to_email
|
||||
msg.attach(MIMEText(render("transactional/user-report.html"), "html"))
|
||||
attachment = MIMEApplication(zipped_contents.read())
|
||||
attachment.add_header(
|
||||
"Content-Disposition", "attachment", filename="user_report.zip"
|
||||
)
|
||||
attachment.add_header("Content-Type", "application/zip")
|
||||
msg.attach(attachment)
|
||||
|
||||
transaction = TransactionalEmail.create(email=to_email, commit=True)
|
||||
sl_sendmail(
|
||||
generate_verp_email(VerpType.transactional, transaction.id),
|
||||
to_email,
|
||||
msg,
|
||||
ignore_smtp_error=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def create_from_job(job: Job) -> Optional[ExportUserDataJob]:
|
||||
user = User.get(job.payload["user_id"])
|
||||
if not user:
|
||||
return None
|
||||
return ExportUserDataJob(user)
|
||||
|
||||
def store_job_in_db(self) -> Optional[Job]:
|
||||
jobs_in_db = (
|
||||
Session.query(Job)
|
||||
.filter(
|
||||
Job.name == config.JOB_SEND_USER_REPORT,
|
||||
Job.payload.op("->")("user_id").cast(sqlalchemy.TEXT)
|
||||
== str(self._user.id),
|
||||
Job.taken.is_(False),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
if jobs_in_db > 0:
|
||||
return None
|
||||
return Job.create(
|
||||
name=config.JOB_SEND_USER_REPORT,
|
||||
payload={"user_id": self._user.id},
|
||||
run_at=arrow.now(),
|
||||
commit=True,
|
||||
)
|
||||
|
|
@ -6,21 +6,14 @@ import time
|
|||
|
||||
import arrow
|
||||
|
||||
from app.config import (
|
||||
JOB_ONBOARDING_1,
|
||||
JOB_ONBOARDING_2,
|
||||
JOB_ONBOARDING_4,
|
||||
JOB_BATCH_IMPORT,
|
||||
JOB_DELETE_ACCOUNT,
|
||||
JOB_DELETE_MAILBOX,
|
||||
JOB_DELETE_DOMAIN,
|
||||
)
|
||||
from app import config
|
||||
from app.db import Session
|
||||
from app.email_utils import (
|
||||
send_email,
|
||||
render,
|
||||
)
|
||||
from app.import_utils import handle_batch_import
|
||||
from app.jobs.export_user_data_job import ExportUserDataJob
|
||||
from app.log import LOG
|
||||
from app.models import User, Job, BatchImport, Mailbox, CustomDomain
|
||||
from server import create_light_app
|
||||
|
|
@ -111,7 +104,7 @@ if __name__ == "__main__":
|
|||
job.taken = True
|
||||
Session.commit()
|
||||
|
||||
if job.name == JOB_ONBOARDING_1:
|
||||
if job.name == config.JOB_ONBOARDING_1:
|
||||
user_id = job.payload.get("user_id")
|
||||
user = User.get(user_id)
|
||||
|
||||
|
|
@ -120,7 +113,7 @@ if __name__ == "__main__":
|
|||
if user and user.notification and user.activated:
|
||||
LOG.d("send onboarding send-from-alias email to user %s", user)
|
||||
onboarding_send_from_alias(user)
|
||||
elif job.name == JOB_ONBOARDING_2:
|
||||
elif job.name == config.JOB_ONBOARDING_2:
|
||||
user_id = job.payload.get("user_id")
|
||||
user = User.get(user_id)
|
||||
|
||||
|
|
@ -129,7 +122,7 @@ if __name__ == "__main__":
|
|||
if user and user.notification and user.activated:
|
||||
LOG.d("send onboarding mailbox email to user %s", user)
|
||||
onboarding_mailbox(user)
|
||||
elif job.name == JOB_ONBOARDING_4:
|
||||
elif job.name == config.JOB_ONBOARDING_4:
|
||||
user_id = job.payload.get("user_id")
|
||||
user = User.get(user_id)
|
||||
|
||||
|
|
@ -139,11 +132,11 @@ if __name__ == "__main__":
|
|||
LOG.d("send onboarding pgp email to user %s", user)
|
||||
onboarding_pgp(user)
|
||||
|
||||
elif job.name == JOB_BATCH_IMPORT:
|
||||
elif job.name == config.JOB_BATCH_IMPORT:
|
||||
batch_import_id = job.payload.get("batch_import_id")
|
||||
batch_import = BatchImport.get(batch_import_id)
|
||||
handle_batch_import(batch_import)
|
||||
elif job.name == JOB_DELETE_ACCOUNT:
|
||||
elif job.name == config.JOB_DELETE_ACCOUNT:
|
||||
user_id = job.payload.get("user_id")
|
||||
user = User.get(user_id)
|
||||
|
||||
|
|
@ -163,7 +156,7 @@ if __name__ == "__main__":
|
|||
render("transactional/account-delete.html"),
|
||||
retries=3,
|
||||
)
|
||||
elif job.name == JOB_DELETE_MAILBOX:
|
||||
elif job.name == config.JOB_DELETE_MAILBOX:
|
||||
mailbox_id = job.payload.get("mailbox_id")
|
||||
mailbox = Mailbox.get(mailbox_id)
|
||||
if not mailbox:
|
||||
|
|
@ -186,7 +179,7 @@ SimpleLogin team.
|
|||
retries=3,
|
||||
)
|
||||
|
||||
elif job.name == JOB_DELETE_DOMAIN:
|
||||
elif job.name == config.JOB_DELETE_DOMAIN:
|
||||
custom_domain_id = job.payload.get("custom_domain_id")
|
||||
custom_domain = CustomDomain.get(custom_domain_id)
|
||||
if not custom_domain:
|
||||
|
|
@ -210,7 +203,10 @@ SimpleLogin team.
|
|||
""",
|
||||
retries=3,
|
||||
)
|
||||
|
||||
if job.name == config.JOB_SEND_USER_REPORT:
|
||||
export_job = ExportUserDataJob.create_from_job(job)
|
||||
if export_job:
|
||||
export_job.run()
|
||||
else:
|
||||
LOG.e("Unknown job name %s", job.name)
|
||||
|
||||
|
|
|
|||
|
|
@ -220,14 +220,15 @@
|
|||
You have linked your Proton account: {{ proton_linked_account }} <br>
|
||||
</div>
|
||||
<a
|
||||
class="btn btn-primary mt-2 proton-button"
|
||||
href="{{ url_for('dashboard.unlink_proton_account') }}"
|
||||
class="btn btn-primary mt-2 proton-button"
|
||||
href="{{ url_for('dashboard.unlink_proton_account') }}"
|
||||
>Unlink account</a>
|
||||
{% else %}
|
||||
<div class="mb-3">
|
||||
You can connect your Proton account with your SimpleLogin one. <br>
|
||||
</div>
|
||||
<a class="btn btn-primary mt-2 proton-button" href="{{ url_for("auth.proton_login", action="link") }}">Connect with Proton</a>
|
||||
<a class="btn btn-primary mt-2 proton-button" href="{{ url_for("auth.proton_login", action="link") }}">Connect
|
||||
with Proton</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -281,7 +282,7 @@
|
|||
{% if current_user.default_random_alias_domain() == domain %} selected {% endif %}
|
||||
{% endif %}
|
||||
>
|
||||
{{ domain }} ({% if is_public %} SimpleLogin domain {% else %} your domain {% endif %})
|
||||
{{ domain }} ({% if is_public %} SimpleLogin domain {% else %} your domain {% endif %})
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
|
@ -591,29 +592,23 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="card">
|
||||
<div class="card-body">
|
||||
<div class="card-title">Data Export</div>
|
||||
<div class="card-title">SimpleLogin data</div>
|
||||
<div class="mb-3">
|
||||
You can download all aliases you have created on SimpleLogin along with other data.
|
||||
As per GDPR law, you can request a copy of your data which are stored on SimpleLogin.
|
||||
A zip file that contains all information will be sent to your SimpleLogin account address.
|
||||
</div>
|
||||
|
||||
<div class="d-flex">
|
||||
<div>
|
||||
<form method="post">
|
||||
<input type="hidden" name="form-name" value="export-data">
|
||||
<button class="btn btn-outline-info">Export Data</button>
|
||||
</form>
|
||||
</div>
|
||||
<div class="ml-5">
|
||||
<form method="post">
|
||||
<input type="hidden" name="form-name" value="export-alias">
|
||||
<button class="btn btn-outline-primary">Export Aliases</button>
|
||||
<input type="hidden" name="form-name" value="send-full-user-report">
|
||||
<button class="btn btn-outline-info">Request your data</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
{% extends "base.html" %}
|
||||
|
||||
{% block content %}
|
||||
{{ render_text("Hi") }}
|
||||
{{ render_text("Please find in the attached zip file a copy of your data which are stored on SimpleLogin. ") }}
|
||||
{{ render_text('Best, <br />SimpleLogin Team.') }}
|
||||
{% endblock %}
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
import zipfile
|
||||
from random import random
|
||||
|
||||
from app.db import Session
|
||||
from app.jobs.export_user_data_job import ExportUserDataJob
|
||||
from app.models import (
|
||||
Contact,
|
||||
Directory,
|
||||
DirectoryMailbox,
|
||||
RefusedEmail,
|
||||
CustomDomain,
|
||||
EmailLog,
|
||||
Alias,
|
||||
)
|
||||
from tests.utils import create_new_user, random_token
|
||||
|
||||
|
||||
def test_model_retrieval_and_serialization():
|
||||
user = create_new_user()
|
||||
job = ExportUserDataJob(user)
|
||||
ExportUserDataJob._model_to_dict(user)
|
||||
|
||||
# Aliases
|
||||
aliases = job._get_aliases()
|
||||
assert len(aliases) == 1
|
||||
ExportUserDataJob._model_to_dict(aliases[0])
|
||||
|
||||
# Mailboxes
|
||||
mailboxes = job._get_mailboxes()
|
||||
assert len(mailboxes) == 1
|
||||
ExportUserDataJob._model_to_dict(mailboxes[0])
|
||||
|
||||
# Contacts
|
||||
alias = aliases[0]
|
||||
contact = Contact.create(
|
||||
website_email=f"marketing-{random()}@example.com",
|
||||
reply_email=f"reply-{random()}@a.b",
|
||||
alias_id=alias.id,
|
||||
user_id=alias.user_id,
|
||||
commit=True,
|
||||
)
|
||||
contacts = job._get_contacts()
|
||||
assert len(contacts) == 1
|
||||
assert contact.id == contacts[0].id
|
||||
ExportUserDataJob._model_to_dict(contacts[0])
|
||||
|
||||
# Directories
|
||||
dir_name = random_token()
|
||||
directory = Directory.create(name=dir_name, user_id=user.id, flush=True)
|
||||
DirectoryMailbox.create(
|
||||
directory_id=directory.id, mailbox_id=user.default_mailbox_id, flush=True
|
||||
)
|
||||
directories = job._get_directories()
|
||||
assert len(directories) == 1
|
||||
assert directory.id == directories[0].id
|
||||
ExportUserDataJob._model_to_dict(directories[0])
|
||||
|
||||
# CustomDomain
|
||||
custom_domain = CustomDomain.create(
|
||||
domain=f"{random()}.com", user_id=user.id, commit=True
|
||||
)
|
||||
domains = job._get_domains()
|
||||
assert len(domains) == 1
|
||||
assert custom_domain.id == domains[0].id
|
||||
ExportUserDataJob._model_to_dict(domains[0])
|
||||
|
||||
# RefusedEmails
|
||||
refused_email = RefusedEmail.create(
|
||||
path=None,
|
||||
full_report_path=f"some/path/{random()}",
|
||||
user_id=alias.user_id,
|
||||
commit=True,
|
||||
)
|
||||
refused_emails = job._get_refused_emails()
|
||||
assert len(refused_emails) == 1
|
||||
assert refused_email.id == refused_emails[0].id
|
||||
ExportUserDataJob._model_to_dict(refused_emails[0])
|
||||
|
||||
# EmailLog
|
||||
email_log = EmailLog.create(
|
||||
user_id=user.id,
|
||||
refused_email_id=refused_email.id,
|
||||
mailbox_id=alias.mailbox.id,
|
||||
contact_id=contact.id,
|
||||
alias_id=alias.id,
|
||||
commit=True,
|
||||
)
|
||||
email_logs = job._get_email_logs()
|
||||
assert len(email_logs) == 1
|
||||
assert email_log.id == email_logs[0].id
|
||||
ExportUserDataJob._model_to_dict(email_logs[0])
|
||||
|
||||
# Get zip
|
||||
memfile = job._build_zip()
|
||||
files_in_zip = set()
|
||||
with zipfile.ZipFile(memfile, "r") as zf:
|
||||
for file_info in zf.infolist():
|
||||
files_in_zip.add(file_info.filename)
|
||||
assert file_info.file_size > 0
|
||||
expected_files_in_zip = set(
|
||||
(
|
||||
"user.json",
|
||||
"aliases.json",
|
||||
"mailboxes.json",
|
||||
"contacts.json",
|
||||
"directories.json",
|
||||
"domains.json",
|
||||
"email_logs.json",
|
||||
# "refused_emails.json",
|
||||
)
|
||||
)
|
||||
assert expected_files_in_zip == files_in_zip
|
||||
|
||||
|
||||
def test_model_retrieval_pagination():
|
||||
user = create_new_user()
|
||||
aliases = Session.query(Alias).filter(Alias.user_id == user.id).all()
|
||||
for _i in range(5):
|
||||
aliases.append(Alias.create_new_random(user))
|
||||
Session.commit()
|
||||
found_aliases = ExportUserDataJob(user)._get_paginated_model(Alias, 2)
|
||||
assert len(found_aliases) == len(aliases)
|
||||
|
||||
|
||||
def test_send_report():
|
||||
user = create_new_user()
|
||||
ExportUserDataJob(user).run()
|
||||
|
||||
|
||||
def test_store_and_retrieve():
|
||||
user = create_new_user()
|
||||
export_job = ExportUserDataJob(user)
|
||||
db_job = export_job.store_job_in_db()
|
||||
assert db_job is not None
|
||||
export_from_from_db = ExportUserDataJob.create_from_job(db_job)
|
||||
assert export_job._user.id == export_from_from_db._user.id
|
||||
|
||||
|
||||
def test_double_store_fails():
|
||||
user = create_new_user()
|
||||
export_job = ExportUserDataJob(user)
|
||||
db_job = export_job.store_job_in_db()
|
||||
assert db_job is not None
|
||||
retry = export_job.store_job_in_db()
|
||||
assert retry is None
|
||||
Loading…
Reference in New Issue