Send aggregated breaches email daily

app/api/serializer.py

@@ -323,7 +323,7 @@ def get_alias_infos_with_pagination_v3(
     elif alias_filter == "disabled":
         q = q.filter(Alias.enabled.is_(False))
     elif alias_filter == "hibp":
-        q = q.filter(Alias.hibp_breaches)
+        q = q.filter(or_(Alias.hibp_breaches_notified_user, Alias.hibp_breaches_not_notified_user))
 
     q = q.order_by(Alias.pinned.desc())
 

app/models.py

@@ -162,9 +162,23 @@ class AliasGeneratorEnum(EnumE):
     uuid = 2  # aliases are generated based on uuid
 
 
+class HibpDataClass(db.Model, ModelMixin):
+    __tablename__ = "hibpdataclass"
+    description = db.Column(db.Text, nullable=False, unique=True, index=True)
+
+    breaches = db.relationship("Hibp", secondary="hibp_hibpdataclass")
+
+    def __repr__(self):
+        return f"<HIBP Data Class {self.id} {self.description}>"
+
+
 class Hibp(db.Model, ModelMixin):
     __tablename__ = "hibp"
     name = db.Column(db.String(), nullable=False, unique=True, index=True)
+    description = db.Column(db.Text)
+    date = db.Column(ArrowType, nullable=True)
+    data_classes = db.relationship("HibpDataClass", secondary="hibp_hibpdataclass")
+
     breached_aliases = db.relationship("Alias", secondary="alias_hibp")
 
     def __repr__(self):
@@ -1062,11 +1076,16 @@ class Alias(db.Model, ModelMixin):
 
     # have I been pwned
     hibp_last_check = db.Column(ArrowType, default=None)
-    hibp_breaches = db.relationship("Hibp", secondary="alias_hibp")
+    hibp_breaches_notified_user = db.relationship("Hibp", secondary="alias_hibp")
+    hibp_breaches_not_notified_user = db.relationship("Hibp", secondary="alias_hibp")
 
     user = db.relationship(User, foreign_keys=[user_id])
     mailbox = db.relationship("Mailbox", lazy="joined")
 
+    @property
+    def hibp_breaches(self):
+        return self.hibp_breaches_notified_user + self.hibp_breaches_not_notified_user
+
     @property
     def mailboxes(self):
         ret = [self.mailbox]
@@ -2061,6 +2080,22 @@ class DomainMailbox(db.Model, ModelMixin):
     )
 
 
+class HibpHibpDataClass(db.Model, ModelMixin):
+    __tablename__ = "hibp_hibpdataclass"
+
+    __table_args__ = (db.UniqueConstraint("hibp_id", "hibp_dataclass_id", name="uq_hibp_hibpdataclass"),)
+
+    hibp_id = db.Column(db.Integer(), db.ForeignKey("hibp.id"))
+    hibp_dataclass_id = db.Column(db.Integer(), db.ForeignKey("hibpdataclass.id"))
+
+    hibp = db.relationship(
+        "Hibp", backref=db.backref("hibp_hibpdataclass", cascade="all, delete-orphan")
+    )
+    hibpdataclass = db.relationship(
+        "HibpDataClass", backref=db.backref("hibp_hibpdataclass", cascade="all, delete-orphan")
+    )
+
+
 _NB_RECOVERY_CODE = 8
 _RECOVERY_CODE_LENGTH = 8
 

cron.py

@@ -56,6 +56,7 @@ from app.models import (
     DeletedAlias,
     DomainDeletedAlias,
     Hibp,
+    HibpDataClass
 )
 from app.utils import sanitize_email
 from server import create_app
@@ -793,14 +794,14 @@ async def _hibp_check(api_key, queue):
 
         if r.status_code == 200:
             # Breaches found
-            alias.hibp_breaches = [
+            breaches = [
                 Hibp.get_by(name=entry["Name"]) for entry in r.json()
             ]
             if len(alias.hibp_breaches) > 0:
                 LOG.w("%s appears in HIBP breaches %s", alias, alias.hibp_breaches)
         elif r.status_code == 404:
             # No breaches found
-            alias.hibp_breaches = []
+            breaches = []
         else:
             LOG.error(
                 "An error occured while checking alias %s: %s - %s",
@@ -810,6 +811,9 @@ async def _hibp_check(api_key, queue):
             )
             return
 
+        alias.hibp_breaches_not_notified_user = [breach for breach in breaches if breach not in alias.hibp_breaches_notified_user]
+        alias.hibp_breaches_notified_user = [breach for breach in breaches if breach not in alias.hibp_breaches_not_notified_user]
+
         alias.hibp_last_check = arrow.utcnow()
         db.session.add(alias)
         db.session.commit()
@@ -829,10 +833,22 @@ async def check_hibp():
         LOG.exception("No HIBP API keys")
         return
 
+    LOG.d("Updating list of known breach types")
+    r = requests.get("https://haveibeenpwned.com/api/v3/dataclasses")
+    for entry in r.json():
+        HibpDataClass.get_or_create(description=entry)
+
+    db.session.commit()
+
     LOG.d("Updating list of known breaches")
     r = requests.get("https://haveibeenpwned.com/api/v3/breaches")
     for entry in r.json():
-        Hibp.get_or_create(name=entry["Name"])
+        hibp_entry = Hibp.get_or_create(name=entry["Name"])
+        hibp_entry.date = arrow.get(entry["BreachDate"])
+        hibp_entry.description = entry["Description"]
+        hibp_entry.data_classes = [
+            HibpDataClass.get_by(description=description) for description in entry["DataClasses"]
+        ]
 
     db.session.commit()
     LOG.d("Updated list of known breaches")
@@ -872,6 +888,41 @@ async def check_hibp():
     LOG.d("Done checking HIBP API for aliases in breaches")
 
 
+def notify_hibp():
+    """
+    Send aggregated email reports for HIBP breaches
+    """
+    for user in User.query.all():
+        new_breaches = Alias.query.filter(Alias.user_id == user.id).filter(Alias.hibp_breaches_not_notified_user).all()
+
+        if len(new_breaches) == 0:
+            return
+
+        LOG.d(f"Send new breaches found email to user {user}")
+
+        send_email(
+            user.email,
+            f"You were in a data breach",
+            render(
+                "transactional/hibp-new-breaches.txt",
+                user=user,
+                breached_aliases=new_breaches
+            ),
+            render(
+                "transactional/hibp-new-breaches.html",
+                user=user,
+                breached_aliases=new_breaches
+            ),
+        )
+
+        for alias in new_breaches:
+            alias.hibp_breaches_notified_user = alias.hibp_breaches_notified_user + alias.hibp_breaches_not_notified_user
+            alias.hibp_breaches_not_notified_user = []
+            db.session.add(alias)
+
+        db.session.commit()
+
+
 if __name__ == "__main__":
     LOG.d("Start running cronjob")
     parser = argparse.ArgumentParser()
@@ -891,6 +942,7 @@ if __name__ == "__main__":
             "delete_old_monitoring",
             "check_custom_domain",
             "check_hibp",
+            "notify_hibp"
         ],
     )
     args = parser.parse_args()
@@ -928,3 +980,6 @@ if __name__ == "__main__":
         elif args.job == "check_hibp":
             LOG.d("Check HIBP")
             asyncio.run(check_hibp())
+        elif args.job == "notify_hibp":
+            LOG.d("Notify users with new HIBP breaches")
+            notify_hibp()

crontab.yml

@@ -58,4 +58,11 @@ jobs:
     shell: /bin/bash
     schedule: "0 18 * * *"
     captureStderr: true
+    concurrencyPolicy: Forbid
+
+  - name: SimpleLogin HIBP report
+    command: python /code/cron.py -j notify_hibp
+    shell: /bin/bash
+    schedule: "0 16 * * *"
+    captureStderr: true
     concurrencyPolicy: Forbid

example.env

@@ -171,5 +171,5 @@ DISABLE_ONBOARDING=true
 # ENABLE_SPAM_ASSASSIN = 1
 
 # Have I Been Pwned
-# HIBP_SCAN_INTERVAL_DAYS = 7
+# HIBP_SCAN_INTERVAL_DAYS = 1
 # HIBP_API_KEYS=[]

migrations/versions/2021_052419_618614d64d1c_.py

@@ -0,0 +1,53 @@
+"""empty message
+
+Revision ID: 618614d64d1c
+Revises: 6cc7f073b358
+Create Date: 2021-05-24 19:44:26.161445
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '618614d64d1c'
+down_revision = '6cc7f073b358'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('hibpdataclass',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
+    sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
+    sa.Column('description', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_hibpdataclass_description'), 'hibpdataclass', ['description'], unique=True)
+    op.create_table('hibp_hibpdataclass',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
+    sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
+    sa.Column('hibp_id', sa.Integer(), nullable=True),
+    sa.Column('hibp_dataclass_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['hibp_dataclass_id'], ['hibpdataclass.id'], ),
+    sa.ForeignKeyConstraint(['hibp_id'], ['hibp.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('hibp_id', 'hibp_dataclass_id', name='uq_hibp_hibpdataclass')
+    )
+    op.add_column('hibp', sa.Column('date', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True))
+    op.add_column('hibp', sa.Column('description', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('hibp', 'description')
+    op.drop_column('hibp', 'date')
+    op.drop_table('hibp_hibpdataclass')
+    op.drop_index(op.f('ix_hibpdataclass_description'), table_name='hibpdataclass')
+    op.drop_table('hibpdataclass')
+    # ### end Alembic commands ###

templates/emails/transactional/hibp-new-breaches.html

@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+
+{% block content %}
+  {{ render_text("Hi") }}
+
+  {%- for alias in breached_aliases %}
+  {%- if loop.index == 11 and loop.length > 10 %}
+  And {{ breached_aliases|length - 10 }} more aliases...
+  {%- elif loop.index > 11 %}
+  {%- else %}
+  {{ render_text("Your alias " + alias.email + " was found in one or more data breaches:") }}
+
+  {%- for breach in alias.hibp_breaches_not_notified_user|sort(attribute='date', reverse=True) %}
+  {%- if loop.index == 4 and loop.length > 4 %}
+  {{ render_text("And " + ((alias.hibp_breaches_not_notified_user|length) - 3)|string + " more data breaches...") }}
+  {%- elif loop.index > 4 %}
+  {%- else %}
+  {{ render_text(breach.name + " (" + breach.date.format('YYYY-MM-DD') + ")") }}
+  {{ render_text(breach.description) }}
+
+  {{ render_text("Breached data: " + breach.data_classes|join(', ', attribute='description')) }}
+  {%- endif %}
+  {%- endfor %}
+  {%- endif %}
+  {%- endfor %}
+
+  {{ render_text("For more information, check <a href='https://haveibeenpwned.com/'>HaveIBeenPwned.com</a>.") }}
+
+  {{ render_text('Best, <br />SimpleLogin Team.') }}
+{% endblock %}

templates/emails/transactional/hibp-new-breaches.txt

@@ -0,0 +1,30 @@
+Hi
+
+{%- for alias in breached_aliases %}
+{%- if loop.index == 11 and loop.length > 10 %}
+And {{ breached_aliases|length - 10 }} more aliases...
+{%- elif loop.index > 11 %}
+{%- else %}
+
+Your alias {{ alias.email }} was found in one or more data breaches:
+
+{%- for breach in alias.hibp_breaches_not_notified_user|sort(attribute='date', reverse=True) %}
+{%- if loop.index == 4 and loop.length > 4 %}
+
+And {{ (alias.hibp_breaches_not_notified_user|length) - 3 }} more data breaches...
+{%- elif loop.index > 4 %}
+{%- else %}
+
+{{ breach.name }} ({{ breach.date.format('YYYY-MM-DD') }})
+{{ breach.description|striptags }}
+
+Breached data: {{ breach.data_classes|join(', ', attribute='description') }}
+{%- endif %}
+{%- endfor %}
+{%- endif %}
+{%- endfor %}
+
+For more information, check https://haveibeenpwned.com/.
+
+Best,
+SimpleLogin Team.