From 1694923e6c91170b0f50dfdf34a702211928da3b Mon Sep 17 00:00:00 2001 From: Sylvia van Os Date: Mon, 24 May 2021 23:55:16 +0200 Subject: [PATCH] Send aggregated breaches email daily --- app/api/serializer.py | 2 +- app/models.py | 37 ++++++++++- cron.py | 61 ++++++++++++++++++- crontab.yml | 7 +++ example.env | 2 +- .../versions/2021_052419_618614d64d1c_.py | 53 ++++++++++++++++ .../transactional/hibp-new-breaches.html | 30 +++++++++ .../transactional/hibp-new-breaches.txt | 30 +++++++++ 8 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 migrations/versions/2021_052419_618614d64d1c_.py create mode 100644 templates/emails/transactional/hibp-new-breaches.html create mode 100644 templates/emails/transactional/hibp-new-breaches.txt diff --git a/app/api/serializer.py b/app/api/serializer.py index 5435a938..2b150311 100644 --- a/app/api/serializer.py +++ b/app/api/serializer.py @@ -323,7 +323,7 @@ def get_alias_infos_with_pagination_v3( elif alias_filter == "disabled": q = q.filter(Alias.enabled.is_(False)) elif alias_filter == "hibp": - q = q.filter(Alias.hibp_breaches) + q = q.filter(or_(Alias.hibp_breaches_notified_user, Alias.hibp_breaches_not_notified_user)) q = q.order_by(Alias.pinned.desc()) diff --git a/app/models.py b/app/models.py index 2ad66141..bae208b9 100644 --- a/app/models.py +++ b/app/models.py @@ -162,9 +162,23 @@ class AliasGeneratorEnum(EnumE): uuid = 2 # aliases are generated based on uuid +class HibpDataClass(db.Model, ModelMixin): + __tablename__ = "hibpdataclass" + description = db.Column(db.Text, nullable=False, unique=True, index=True) + + breaches = db.relationship("Hibp", secondary="hibp_hibpdataclass") + + def __repr__(self): + return f"" + + class Hibp(db.Model, ModelMixin): __tablename__ = "hibp" name = db.Column(db.String(), nullable=False, unique=True, index=True) + description = db.Column(db.Text) + date = db.Column(ArrowType, nullable=True) + data_classes = db.relationship("HibpDataClass", secondary="hibp_hibpdataclass") + breached_aliases = db.relationship("Alias", secondary="alias_hibp") def __repr__(self): @@ -1062,11 +1076,16 @@ class Alias(db.Model, ModelMixin): # have I been pwned hibp_last_check = db.Column(ArrowType, default=None) - hibp_breaches = db.relationship("Hibp", secondary="alias_hibp") + hibp_breaches_notified_user = db.relationship("Hibp", secondary="alias_hibp") + hibp_breaches_not_notified_user = db.relationship("Hibp", secondary="alias_hibp") user = db.relationship(User, foreign_keys=[user_id]) mailbox = db.relationship("Mailbox", lazy="joined") + @property + def hibp_breaches(self): + return self.hibp_breaches_notified_user + self.hibp_breaches_not_notified_user + @property def mailboxes(self): ret = [self.mailbox] @@ -2061,6 +2080,22 @@ class DomainMailbox(db.Model, ModelMixin): ) +class HibpHibpDataClass(db.Model, ModelMixin): + __tablename__ = "hibp_hibpdataclass" + + __table_args__ = (db.UniqueConstraint("hibp_id", "hibp_dataclass_id", name="uq_hibp_hibpdataclass"),) + + hibp_id = db.Column(db.Integer(), db.ForeignKey("hibp.id")) + hibp_dataclass_id = db.Column(db.Integer(), db.ForeignKey("hibpdataclass.id")) + + hibp = db.relationship( + "Hibp", backref=db.backref("hibp_hibpdataclass", cascade="all, delete-orphan") + ) + hibpdataclass = db.relationship( + "HibpDataClass", backref=db.backref("hibp_hibpdataclass", cascade="all, delete-orphan") + ) + + _NB_RECOVERY_CODE = 8 _RECOVERY_CODE_LENGTH = 8 diff --git a/cron.py b/cron.py index 2af1cb1f..c1651fe7 100644 --- a/cron.py +++ b/cron.py @@ -56,6 +56,7 @@ from app.models import ( DeletedAlias, DomainDeletedAlias, Hibp, + HibpDataClass ) from app.utils import sanitize_email from server import create_app @@ -793,14 +794,14 @@ async def _hibp_check(api_key, queue): if r.status_code == 200: # Breaches found - alias.hibp_breaches = [ + breaches = [ Hibp.get_by(name=entry["Name"]) for entry in r.json() ] if len(alias.hibp_breaches) > 0: LOG.w("%s appears in HIBP breaches %s", alias, alias.hibp_breaches) elif r.status_code == 404: # No breaches found - alias.hibp_breaches = [] + breaches = [] else: LOG.error( "An error occured while checking alias %s: %s - %s", @@ -810,6 +811,9 @@ async def _hibp_check(api_key, queue): ) return + alias.hibp_breaches_not_notified_user = [breach for breach in breaches if breach not in alias.hibp_breaches_notified_user] + alias.hibp_breaches_notified_user = [breach for breach in breaches if breach not in alias.hibp_breaches_not_notified_user] + alias.hibp_last_check = arrow.utcnow() db.session.add(alias) db.session.commit() @@ -829,10 +833,22 @@ async def check_hibp(): LOG.exception("No HIBP API keys") return + LOG.d("Updating list of known breach types") + r = requests.get("https://haveibeenpwned.com/api/v3/dataclasses") + for entry in r.json(): + HibpDataClass.get_or_create(description=entry) + + db.session.commit() + LOG.d("Updating list of known breaches") r = requests.get("https://haveibeenpwned.com/api/v3/breaches") for entry in r.json(): - Hibp.get_or_create(name=entry["Name"]) + hibp_entry = Hibp.get_or_create(name=entry["Name"]) + hibp_entry.date = arrow.get(entry["BreachDate"]) + hibp_entry.description = entry["Description"] + hibp_entry.data_classes = [ + HibpDataClass.get_by(description=description) for description in entry["DataClasses"] + ] db.session.commit() LOG.d("Updated list of known breaches") @@ -872,6 +888,41 @@ async def check_hibp(): LOG.d("Done checking HIBP API for aliases in breaches") +def notify_hibp(): + """ + Send aggregated email reports for HIBP breaches + """ + for user in User.query.all(): + new_breaches = Alias.query.filter(Alias.user_id == user.id).filter(Alias.hibp_breaches_not_notified_user).all() + + if len(new_breaches) == 0: + return + + LOG.d(f"Send new breaches found email to user {user}") + + send_email( + user.email, + f"You were in a data breach", + render( + "transactional/hibp-new-breaches.txt", + user=user, + breached_aliases=new_breaches + ), + render( + "transactional/hibp-new-breaches.html", + user=user, + breached_aliases=new_breaches + ), + ) + + for alias in new_breaches: + alias.hibp_breaches_notified_user = alias.hibp_breaches_notified_user + alias.hibp_breaches_not_notified_user + alias.hibp_breaches_not_notified_user = [] + db.session.add(alias) + + db.session.commit() + + if __name__ == "__main__": LOG.d("Start running cronjob") parser = argparse.ArgumentParser() @@ -891,6 +942,7 @@ if __name__ == "__main__": "delete_old_monitoring", "check_custom_domain", "check_hibp", + "notify_hibp" ], ) args = parser.parse_args() @@ -928,3 +980,6 @@ if __name__ == "__main__": elif args.job == "check_hibp": LOG.d("Check HIBP") asyncio.run(check_hibp()) + elif args.job == "notify_hibp": + LOG.d("Notify users with new HIBP breaches") + notify_hibp() \ No newline at end of file diff --git a/crontab.yml b/crontab.yml index d22f14c2..7439ab83 100644 --- a/crontab.yml +++ b/crontab.yml @@ -58,4 +58,11 @@ jobs: shell: /bin/bash schedule: "0 18 * * *" captureStderr: true + concurrencyPolicy: Forbid + + - name: SimpleLogin HIBP report + command: python /code/cron.py -j notify_hibp + shell: /bin/bash + schedule: "0 16 * * *" + captureStderr: true concurrencyPolicy: Forbid \ No newline at end of file diff --git a/example.env b/example.env index a2d1963d..5f20110d 100644 --- a/example.env +++ b/example.env @@ -171,5 +171,5 @@ DISABLE_ONBOARDING=true # ENABLE_SPAM_ASSASSIN = 1 # Have I Been Pwned -# HIBP_SCAN_INTERVAL_DAYS = 7 +# HIBP_SCAN_INTERVAL_DAYS = 1 # HIBP_API_KEYS=[] \ No newline at end of file diff --git a/migrations/versions/2021_052419_618614d64d1c_.py b/migrations/versions/2021_052419_618614d64d1c_.py new file mode 100644 index 00000000..31cf5d58 --- /dev/null +++ b/migrations/versions/2021_052419_618614d64d1c_.py @@ -0,0 +1,53 @@ +"""empty message + +Revision ID: 618614d64d1c +Revises: 6cc7f073b358 +Create Date: 2021-05-24 19:44:26.161445 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '618614d64d1c' +down_revision = '6cc7f073b358' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('hibpdataclass', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('description', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_hibpdataclass_description'), 'hibpdataclass', ['description'], unique=True) + op.create_table('hibp_hibpdataclass', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('hibp_id', sa.Integer(), nullable=True), + sa.Column('hibp_dataclass_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['hibp_dataclass_id'], ['hibpdataclass.id'], ), + sa.ForeignKeyConstraint(['hibp_id'], ['hibp.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('hibp_id', 'hibp_dataclass_id', name='uq_hibp_hibpdataclass') + ) + op.add_column('hibp', sa.Column('date', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True)) + op.add_column('hibp', sa.Column('description', sa.Text(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('hibp', 'description') + op.drop_column('hibp', 'date') + op.drop_table('hibp_hibpdataclass') + op.drop_index(op.f('ix_hibpdataclass_description'), table_name='hibpdataclass') + op.drop_table('hibpdataclass') + # ### end Alembic commands ### diff --git a/templates/emails/transactional/hibp-new-breaches.html b/templates/emails/transactional/hibp-new-breaches.html new file mode 100644 index 00000000..9b0a6737 --- /dev/null +++ b/templates/emails/transactional/hibp-new-breaches.html @@ -0,0 +1,30 @@ +{% extends "base.html" %} + +{% block content %} + {{ render_text("Hi") }} + + {%- for alias in breached_aliases %} + {%- if loop.index == 11 and loop.length > 10 %} + And {{ breached_aliases|length - 10 }} more aliases... + {%- elif loop.index > 11 %} + {%- else %} + {{ render_text("Your alias " + alias.email + " was found in one or more data breaches:") }} + + {%- for breach in alias.hibp_breaches_not_notified_user|sort(attribute='date', reverse=True) %} + {%- if loop.index == 4 and loop.length > 4 %} + {{ render_text("And " + ((alias.hibp_breaches_not_notified_user|length) - 3)|string + " more data breaches...") }} + {%- elif loop.index > 4 %} + {%- else %} + {{ render_text(breach.name + " (" + breach.date.format('YYYY-MM-DD') + ")") }} + {{ render_text(breach.description) }} + + {{ render_text("Breached data: " + breach.data_classes|join(', ', attribute='description')) }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endfor %} + + {{ render_text("For more information, check HaveIBeenPwned.com.") }} + + {{ render_text('Best,
SimpleLogin Team.') }} +{% endblock %} diff --git a/templates/emails/transactional/hibp-new-breaches.txt b/templates/emails/transactional/hibp-new-breaches.txt new file mode 100644 index 00000000..d9e81cc5 --- /dev/null +++ b/templates/emails/transactional/hibp-new-breaches.txt @@ -0,0 +1,30 @@ +Hi + +{%- for alias in breached_aliases %} +{%- if loop.index == 11 and loop.length > 10 %} +And {{ breached_aliases|length - 10 }} more aliases... +{%- elif loop.index > 11 %} +{%- else %} + +Your alias {{ alias.email }} was found in one or more data breaches: + +{%- for breach in alias.hibp_breaches_not_notified_user|sort(attribute='date', reverse=True) %} +{%- if loop.index == 4 and loop.length > 4 %} + +And {{ (alias.hibp_breaches_not_notified_user|length) - 3 }} more data breaches... +{%- elif loop.index > 4 %} +{%- else %} + +{{ breach.name }} ({{ breach.date.format('YYYY-MM-DD') }}) +{{ breach.description|striptags }} + +Breached data: {{ breach.data_classes|join(', ', attribute='description') }} +{%- endif %} +{%- endfor %} +{%- endif %} +{%- endfor %} + +For more information, check https://haveibeenpwned.com/. + +Best, +SimpleLogin Team.