From e9e863807cf1ba69db5dd16ed6ceda18957091fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Sat, 29 Jul 2023 10:03:31 +0200 Subject: [PATCH] Add missing indexes (#1824) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Rate limit the sudo route * Add missing indexes * Updated index * Update index creation to run with concurrent * With autocommit block --------- Co-authored-by: Adrià Casajús --- app/models.py | 21 ++++++++-- cron.py | 4 +- crontab.yml | 6 --- .../versions/2023_072819_01827104004b_.py | 42 +++++++++++++++++++ 4 files changed, 63 insertions(+), 10 deletions(-) create mode 100644 migrations/versions/2023_072819_01827104004b_.py diff --git a/app/models.py b/app/models.py index 3b4df673..faec2e19 100644 --- a/app/models.py +++ b/app/models.py @@ -341,7 +341,7 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): sa.Boolean, default=True, nullable=False, server_default="1" ) - activated = sa.Column(sa.Boolean, default=False, nullable=False) + activated = sa.Column(sa.Boolean, default=False, nullable=False, index=True) # an account can be disabled if having harmful behavior disabled = sa.Column(sa.Boolean, default=False, nullable=False, server_default="0") @@ -411,7 +411,10 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): ) referral_id = sa.Column( - sa.ForeignKey("referral.id", ondelete="SET NULL"), nullable=True, default=None + sa.ForeignKey("referral.id", ondelete="SET NULL"), + nullable=True, + default=None, + index=True, ) referral = orm.relationship("Referral", foreign_keys=[referral_id]) @@ -534,6 +537,12 @@ class User(Base, ModelMixin, UserMixin, PasswordOracle): nullable=False, ) + __table_args__ = ( + sa.Index( + "ix_users_activated_trial_end_lifetime", activated, trial_end, lifetime + ), + ) + @property def directory_quota(self): return min( @@ -1445,7 +1454,7 @@ class Alias(Base, ModelMixin): ) # have I been pwned - hibp_last_check = sa.Column(ArrowType, default=None) + hibp_last_check = sa.Column(ArrowType, default=None, index=True) hibp_breaches = orm.relationship("Hibp", secondary="alias_hibp") # to use Postgres full text search. Only applied on "note" column for now @@ -2928,6 +2937,8 @@ class Monitoring(Base, ModelMixin): active_queue = sa.Column(sa.Integer, nullable=False) deferred_queue = sa.Column(sa.Integer, nullable=False) + __table_args__ = (Index("ix_monitoring_created_at", "created_at"),) + class BatchImport(Base, ModelMixin): __tablename__ = "batch_import" @@ -3053,6 +3064,8 @@ class Bounce(Base, ModelMixin): email = sa.Column(sa.String(256), nullable=False, index=True) info = sa.Column(sa.Text, nullable=True) + __table_args__ = (sa.Index("ix_bounce_created_at", "created_at"),) + class TransactionalEmail(Base, ModelMixin): """Storing all email addresses that receive transactional emails, including account email and mailboxes. @@ -3062,6 +3075,8 @@ class TransactionalEmail(Base, ModelMixin): __tablename__ = "transactional_email" email = sa.Column(sa.String(256), nullable=False, unique=False) + __table_args__ = (sa.Index("ix_transactional_email_created_at", "created_at"),) + class Payout(Base, ModelMixin): """Referral payouts""" diff --git a/cron.py b/cron.py index 75e8ed78..9d307a59 100644 --- a/cron.py +++ b/cron.py @@ -104,7 +104,9 @@ def delete_logs(): def delete_refused_emails(): - for refused_email in RefusedEmail.filter_by(deleted=False).all(): + for refused_email in ( + RefusedEmail.filter_by(deleted=False).order_by(RefusedEmail.id).all() + ): if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now(): LOG.d("Delete refused email %s", refused_email) if refused_email.path: diff --git a/crontab.yml b/crontab.yml index ec5a257a..ba47303f 100644 --- a/crontab.yml +++ b/crontab.yml @@ -35,12 +35,6 @@ jobs: schedule: "0 12 * * *" captureStderr: true - - name: SimpleLogin Sanity Check - command: python /code/cron.py -j sanity_check - shell: /bin/bash - schedule: "0 2 * * *" - captureStderr: true - - name: SimpleLogin Delete Old Monitoring records command: python /code/cron.py -j delete_old_monitoring shell: /bin/bash diff --git a/migrations/versions/2023_072819_01827104004b_.py b/migrations/versions/2023_072819_01827104004b_.py new file mode 100644 index 00000000..7030508c --- /dev/null +++ b/migrations/versions/2023_072819_01827104004b_.py @@ -0,0 +1,42 @@ +"""empty message + +Revision ID: 01827104004b +Revises: 2634b41f54db +Create Date: 2023-07-28 19:39:28.675490 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '01827104004b' +down_revision = '2634b41f54db' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.get_context().autocommit_block(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('ix_alias_hibp_last_check'), 'alias', ['hibp_last_check'], unique=False, postgresql_concurrently=True) + op.create_index('ix_bounce_created_at', 'bounce', ['created_at'], unique=False, postgresql_concurrently=True) + op.create_index('ix_monitoring_created_at', 'monitoring', ['created_at'], unique=False, postgresql_concurrently=True) + op.create_index('ix_transactional_email_created_at', 'transactional_email', ['created_at'], unique=False, postgresql_concurrently=True) + op.create_index(op.f('ix_users_activated'), 'users', ['activated'], unique=False, postgresql_concurrently=True) + op.create_index('ix_users_activated_trial_end_lifetime', 'users', ['activated', 'trial_end', 'lifetime'], unique=False, postgresql_concurrently=True) + op.create_index(op.f('ix_users_referral_id'), 'users', ['referral_id'], unique=False, postgresql_concurrently=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_users_referral_id'), table_name='users') + op.drop_index('ix_users_activated_trial_end_lifetime', table_name='users') + op.drop_index(op.f('ix_users_activated'), table_name='users') + op.drop_index('ix_transactional_email_created_at', table_name='transactional_email') + op.drop_index('ix_monitoring_created_at', table_name='monitoring') + op.drop_index('ix_bounce_created_at', table_name='bounce') + op.drop_index(op.f('ix_alias_hibp_last_check'), table_name='alias') + # ### end Alembic commands ###