From e688f04d6b12b18fdd1948837592c9ceec9266d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Tue, 7 Jun 2022 10:45:04 +0200 Subject: [PATCH] Send full user report asynchronously on request (#1029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Send full user report asynchronously * Fix test * Filter some fields before exporting * Fix: Domain -> CustomDomain * format settings html * not include RefusedEmail as they are not usable by user and are automatically deleted * send the export to the user email * change email and setting wording * fix user can only export data once * remove alias export section * remove unused import * fix flake8 Co-authored-by: Adrià Casajús Co-authored-by: Son --- app/config.py | 1 + app/dashboard/views/setting.py | 13 +- app/jobs/__init__.py | 0 app/jobs/export_user_data_job.py | 177 ++++++++++++++++++ job_runner.py | 30 ++- templates/dashboard/setting.html | 27 ++- .../emails/transactional/user-report.html | 7 + tests/jobs/__init__.py | 0 tests/jobs/test_export_user_data_job.py | 145 ++++++++++++++ 9 files changed, 363 insertions(+), 37 deletions(-) create mode 100644 app/jobs/__init__.py create mode 100644 app/jobs/export_user_data_job.py create mode 100644 templates/emails/transactional/user-report.html create mode 100644 tests/jobs/__init__.py create mode 100644 tests/jobs/test_export_user_data_job.py diff --git a/app/config.py b/app/config.py index d88708e3..46bce9ac 100644 --- a/app/config.py +++ b/app/config.py @@ -263,6 +263,7 @@ JOB_BATCH_IMPORT = "batch-import" JOB_DELETE_ACCOUNT = "delete-account" JOB_DELETE_MAILBOX = "delete-mailbox" JOB_DELETE_DOMAIN = "delete-domain" +JOB_SEND_USER_REPORT = "send-user-report" # for pagination PAGE_LIMIT = 20 diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index 0f4e1c42..dfcd84b7 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -29,6 +29,7 @@ from app.email_utils import ( personal_email_already_used, ) from app.errors import ProtonPartnerNotSetUp +from app.jobs.export_user_data_job import ExportUserDataJob from app.log import LOG from app.models import ( BlockBehaviourEnum, @@ -348,10 +349,14 @@ def setting(): Session.commit() flash("Your preference has been updated", "success") return redirect(url_for("dashboard.setting")) - elif request.form.get("form-name") == "export-data": - return redirect(url_for("api.export_data")) - elif request.form.get("form-name") == "export-alias": - return redirect(url_for("api.export_aliases")) + elif request.form.get("form-name") == "send-full-user-report": + if ExportUserDataJob(current_user).store_job_in_db(): + flash( + "You will receive your SimpleLogin data via email shortly", + "success", + ) + else: + flash("An export of your data is currently in progress", "error") manual_sub = ManualSubscription.get_by(user_id=current_user.id) apple_sub = AppleSubscription.get_by(user_id=current_user.id) diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/jobs/export_user_data_job.py b/app/jobs/export_user_data_job.py new file mode 100644 index 00000000..1613c2a1 --- /dev/null +++ b/app/jobs/export_user_data_job.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import json +import zipfile +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from io import BytesIO +from typing import List, Dict, Optional + +import arrow +import sqlalchemy + +from app import config +from app.db import Session +from app.email import headers +from app.email_utils import generate_verp_email, render +from app.mail_sender import sl_sendmail +from app.models import ( + Alias, + Contact, + Mailbox, + Directory, + EmailLog, + CustomDomain, + RefusedEmail, + Base, + User, + EnumE, + TransactionalEmail, + VerpType, + Job, +) + + +class ExportUserDataJob: + + REMOVE_FIELDS = { + "User": ("otp_secret",), + "Alias": ("ts_vector", "transfer_token", "hibp_last_check"), + "CustomDomain": ("ownership_txt_token",), + } + + def __init__(self, user: User): + self._user: User = user + + def _get_paginated_model(self, model_class, page_size=50) -> List: + objects = [] + page = 0 + db_objects = [] + while page == 0 or len(db_objects) == page_size: + db_objects = ( + Session.query(model_class) + .filter(model_class.user_id == self._user.id) + .order_by(model_class.id) + .limit(page_size) + .offset(page * page_size) + .all() + ) + objects.extend(db_objects) + page += 1 + return objects + + def _get_aliases(self) -> List[Alias]: + return self._get_paginated_model(Alias) + + def _get_mailboxes(self) -> List[Mailbox]: + return self._get_paginated_model(Mailbox) + + def _get_contacts(self) -> List[Contact]: + return self._get_paginated_model(Contact) + + def _get_directories(self) -> List[Directory]: + return self._get_paginated_model(Directory) + + def _get_email_logs(self) -> List[EmailLog]: + return self._get_paginated_model(EmailLog) + + def _get_domains(self) -> List[CustomDomain]: + return self._get_paginated_model(CustomDomain) + + def _get_refused_emails(self) -> List[RefusedEmail]: + return self._get_paginated_model(RefusedEmail) + + @classmethod + def _model_to_dict(cls, object: Base) -> Dict: + data = {} + fields_to_filter = cls.REMOVE_FIELDS.get(object.__class__.__name__, ()) + for column in object.__table__.columns: + if column.name in fields_to_filter: + continue + value = getattr(object, column.name) + if isinstance(value, arrow.Arrow): + value = value.isoformat() + if issubclass(value.__class__, EnumE): + value = value.value + data[column.name] = value + return data + + def _build_zip(self) -> BytesIO: + memfile = BytesIO() + with zipfile.ZipFile(memfile, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr( + "user.json", json.dumps(ExportUserDataJob._model_to_dict(self._user)) + ) + for model_name, get_models in [ + ("aliases", self._get_aliases), + ("mailboxes", self._get_mailboxes), + ("contacts", self._get_contacts), + ("directories", self._get_directories), + ("domains", self._get_domains), + ("email_logs", self._get_email_logs), + # not include RefusedEmail as they are not usable by user and are automatically deleted + # ("refused_emails", self._get_refused_emails), + ]: + model_objs = get_models() + data = json.dumps( + [ + ExportUserDataJob._model_to_dict(model_obj) + for model_obj in model_objs + ] + ) + zf.writestr(f"{model_name}.json", data) + memfile.seek(0) + return memfile + + def run(self): + zipped_contents = self._build_zip() + + to_email = self._user.email + + msg = MIMEMultipart() + msg[headers.SUBJECT] = "Your SimpleLogin data" + msg[headers.FROM] = f'"SimpleLogin (noreply)" <{config.NOREPLY}>' + msg[headers.TO] = to_email + msg.attach(MIMEText(render("transactional/user-report.html"), "html")) + attachment = MIMEApplication(zipped_contents.read()) + attachment.add_header( + "Content-Disposition", "attachment", filename="user_report.zip" + ) + attachment.add_header("Content-Type", "application/zip") + msg.attach(attachment) + + transaction = TransactionalEmail.create(email=to_email, commit=True) + sl_sendmail( + generate_verp_email(VerpType.transactional, transaction.id), + to_email, + msg, + ignore_smtp_error=False, + ) + + @staticmethod + def create_from_job(job: Job) -> Optional[ExportUserDataJob]: + user = User.get(job.payload["user_id"]) + if not user: + return None + return ExportUserDataJob(user) + + def store_job_in_db(self) -> Optional[Job]: + jobs_in_db = ( + Session.query(Job) + .filter( + Job.name == config.JOB_SEND_USER_REPORT, + Job.payload.op("->")("user_id").cast(sqlalchemy.TEXT) + == str(self._user.id), + Job.taken.is_(False), + ) + .count() + ) + if jobs_in_db > 0: + return None + return Job.create( + name=config.JOB_SEND_USER_REPORT, + payload={"user_id": self._user.id}, + run_at=arrow.now(), + commit=True, + ) diff --git a/job_runner.py b/job_runner.py index b7f1475d..3a726e24 100644 --- a/job_runner.py +++ b/job_runner.py @@ -6,21 +6,14 @@ import time import arrow -from app.config import ( - JOB_ONBOARDING_1, - JOB_ONBOARDING_2, - JOB_ONBOARDING_4, - JOB_BATCH_IMPORT, - JOB_DELETE_ACCOUNT, - JOB_DELETE_MAILBOX, - JOB_DELETE_DOMAIN, -) +from app import config from app.db import Session from app.email_utils import ( send_email, render, ) from app.import_utils import handle_batch_import +from app.jobs.export_user_data_job import ExportUserDataJob from app.log import LOG from app.models import User, Job, BatchImport, Mailbox, CustomDomain from server import create_light_app @@ -111,7 +104,7 @@ if __name__ == "__main__": job.taken = True Session.commit() - if job.name == JOB_ONBOARDING_1: + if job.name == config.JOB_ONBOARDING_1: user_id = job.payload.get("user_id") user = User.get(user_id) @@ -120,7 +113,7 @@ if __name__ == "__main__": if user and user.notification and user.activated: LOG.d("send onboarding send-from-alias email to user %s", user) onboarding_send_from_alias(user) - elif job.name == JOB_ONBOARDING_2: + elif job.name == config.JOB_ONBOARDING_2: user_id = job.payload.get("user_id") user = User.get(user_id) @@ -129,7 +122,7 @@ if __name__ == "__main__": if user and user.notification and user.activated: LOG.d("send onboarding mailbox email to user %s", user) onboarding_mailbox(user) - elif job.name == JOB_ONBOARDING_4: + elif job.name == config.JOB_ONBOARDING_4: user_id = job.payload.get("user_id") user = User.get(user_id) @@ -139,11 +132,11 @@ if __name__ == "__main__": LOG.d("send onboarding pgp email to user %s", user) onboarding_pgp(user) - elif job.name == JOB_BATCH_IMPORT: + elif job.name == config.JOB_BATCH_IMPORT: batch_import_id = job.payload.get("batch_import_id") batch_import = BatchImport.get(batch_import_id) handle_batch_import(batch_import) - elif job.name == JOB_DELETE_ACCOUNT: + elif job.name == config.JOB_DELETE_ACCOUNT: user_id = job.payload.get("user_id") user = User.get(user_id) @@ -163,7 +156,7 @@ if __name__ == "__main__": render("transactional/account-delete.html"), retries=3, ) - elif job.name == JOB_DELETE_MAILBOX: + elif job.name == config.JOB_DELETE_MAILBOX: mailbox_id = job.payload.get("mailbox_id") mailbox = Mailbox.get(mailbox_id) if not mailbox: @@ -186,7 +179,7 @@ SimpleLogin team. retries=3, ) - elif job.name == JOB_DELETE_DOMAIN: + elif job.name == config.JOB_DELETE_DOMAIN: custom_domain_id = job.payload.get("custom_domain_id") custom_domain = CustomDomain.get(custom_domain_id) if not custom_domain: @@ -210,7 +203,10 @@ SimpleLogin team. """, retries=3, ) - + if job.name == config.JOB_SEND_USER_REPORT: + export_job = ExportUserDataJob.create_from_job(job) + if export_job: + export_job.run() else: LOG.e("Unknown job name %s", job.name) diff --git a/templates/dashboard/setting.html b/templates/dashboard/setting.html index 8e2d0ab2..55b6ac03 100644 --- a/templates/dashboard/setting.html +++ b/templates/dashboard/setting.html @@ -220,14 +220,15 @@ You have linked your Proton account: {{ proton_linked_account }}
Unlink account {% else %}
You can connect your Proton account with your SimpleLogin one.
- Connect with Proton + Connect + with Proton {% endif %} @@ -281,7 +282,7 @@ {% if current_user.default_random_alias_domain() == domain %} selected {% endif %} {% endif %} > - {{ domain }} ({% if is_public %} SimpleLogin domain {% else %} your domain {% endif %}) + {{ domain }} ({% if is_public %} SimpleLogin domain {% else %} your domain {% endif %}) {% endfor %} @@ -591,29 +592,23 @@ +
-
Data Export
+
SimpleLogin data
- You can download all aliases you have created on SimpleLogin along with other data. + As per GDPR law, you can request a copy of your data which are stored on SimpleLogin. + A zip file that contains all information will be sent to your SimpleLogin account address.
- - -
-
-
-
- - + +
- -
diff --git a/templates/emails/transactional/user-report.html b/templates/emails/transactional/user-report.html new file mode 100644 index 00000000..f656b862 --- /dev/null +++ b/templates/emails/transactional/user-report.html @@ -0,0 +1,7 @@ +{% extends "base.html" %} + +{% block content %} + {{ render_text("Hi") }} + {{ render_text("Please find in the attached zip file a copy of your data which are stored on SimpleLogin. ") }} + {{ render_text('Best,
SimpleLogin Team.') }} +{% endblock %} diff --git a/tests/jobs/__init__.py b/tests/jobs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/jobs/test_export_user_data_job.py b/tests/jobs/test_export_user_data_job.py new file mode 100644 index 00000000..763b1c0a --- /dev/null +++ b/tests/jobs/test_export_user_data_job.py @@ -0,0 +1,145 @@ +import zipfile +from random import random + +from app.db import Session +from app.jobs.export_user_data_job import ExportUserDataJob +from app.models import ( + Contact, + Directory, + DirectoryMailbox, + RefusedEmail, + CustomDomain, + EmailLog, + Alias, +) +from tests.utils import create_new_user, random_token + + +def test_model_retrieval_and_serialization(): + user = create_new_user() + job = ExportUserDataJob(user) + ExportUserDataJob._model_to_dict(user) + + # Aliases + aliases = job._get_aliases() + assert len(aliases) == 1 + ExportUserDataJob._model_to_dict(aliases[0]) + + # Mailboxes + mailboxes = job._get_mailboxes() + assert len(mailboxes) == 1 + ExportUserDataJob._model_to_dict(mailboxes[0]) + + # Contacts + alias = aliases[0] + contact = Contact.create( + website_email=f"marketing-{random()}@example.com", + reply_email=f"reply-{random()}@a.b", + alias_id=alias.id, + user_id=alias.user_id, + commit=True, + ) + contacts = job._get_contacts() + assert len(contacts) == 1 + assert contact.id == contacts[0].id + ExportUserDataJob._model_to_dict(contacts[0]) + + # Directories + dir_name = random_token() + directory = Directory.create(name=dir_name, user_id=user.id, flush=True) + DirectoryMailbox.create( + directory_id=directory.id, mailbox_id=user.default_mailbox_id, flush=True + ) + directories = job._get_directories() + assert len(directories) == 1 + assert directory.id == directories[0].id + ExportUserDataJob._model_to_dict(directories[0]) + + # CustomDomain + custom_domain = CustomDomain.create( + domain=f"{random()}.com", user_id=user.id, commit=True + ) + domains = job._get_domains() + assert len(domains) == 1 + assert custom_domain.id == domains[0].id + ExportUserDataJob._model_to_dict(domains[0]) + + # RefusedEmails + refused_email = RefusedEmail.create( + path=None, + full_report_path=f"some/path/{random()}", + user_id=alias.user_id, + commit=True, + ) + refused_emails = job._get_refused_emails() + assert len(refused_emails) == 1 + assert refused_email.id == refused_emails[0].id + ExportUserDataJob._model_to_dict(refused_emails[0]) + + # EmailLog + email_log = EmailLog.create( + user_id=user.id, + refused_email_id=refused_email.id, + mailbox_id=alias.mailbox.id, + contact_id=contact.id, + alias_id=alias.id, + commit=True, + ) + email_logs = job._get_email_logs() + assert len(email_logs) == 1 + assert email_log.id == email_logs[0].id + ExportUserDataJob._model_to_dict(email_logs[0]) + + # Get zip + memfile = job._build_zip() + files_in_zip = set() + with zipfile.ZipFile(memfile, "r") as zf: + for file_info in zf.infolist(): + files_in_zip.add(file_info.filename) + assert file_info.file_size > 0 + expected_files_in_zip = set( + ( + "user.json", + "aliases.json", + "mailboxes.json", + "contacts.json", + "directories.json", + "domains.json", + "email_logs.json", + # "refused_emails.json", + ) + ) + assert expected_files_in_zip == files_in_zip + + +def test_model_retrieval_pagination(): + user = create_new_user() + aliases = Session.query(Alias).filter(Alias.user_id == user.id).all() + for _i in range(5): + aliases.append(Alias.create_new_random(user)) + Session.commit() + found_aliases = ExportUserDataJob(user)._get_paginated_model(Alias, 2) + assert len(found_aliases) == len(aliases) + + +def test_send_report(): + user = create_new_user() + ExportUserDataJob(user).run() + + +def test_store_and_retrieve(): + user = create_new_user() + export_job = ExportUserDataJob(user) + db_job = export_job.store_job_in_db() + assert db_job is not None + export_from_from_db = ExportUserDataJob.create_from_job(db_job) + assert export_job._user.id == export_from_from_db._user.id + + +def test_double_store_fails(): + user = create_new_user() + export_job = ExportUserDataJob(user) + db_job = export_job.store_job_in_db() + assert db_job is not None + retry = export_job.store_job_in_db() + assert retry is None