From 5c9abdeff1d0cf3e14d84c5ae298e6a5beccaf18 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 27 Sep 2022 03:08:19 +0200 Subject: Add retention policy for cached content and media (#19232) --- app/lib/redis_configuration.rb | 6 ++-- app/lib/vacuum.rb | 3 ++ app/lib/vacuum/access_tokens_vacuum.rb | 18 ++++++++++ app/lib/vacuum/backups_vacuum.rb | 25 ++++++++++++++ app/lib/vacuum/feeds_vacuum.rb | 34 +++++++++++++++++++ app/lib/vacuum/media_attachments_vacuum.rb | 40 ++++++++++++++++++++++ app/lib/vacuum/preview_cards_vacuum.rb | 39 +++++++++++++++++++++ app/lib/vacuum/statuses_vacuum.rb | 54 ++++++++++++++++++++++++++++++ app/lib/vacuum/system_keys_vacuum.rb | 13 +++++++ 9 files changed, 228 insertions(+), 4 deletions(-) create mode 100644 app/lib/vacuum.rb create mode 100644 app/lib/vacuum/access_tokens_vacuum.rb create mode 100644 app/lib/vacuum/backups_vacuum.rb create mode 100644 app/lib/vacuum/feeds_vacuum.rb create mode 100644 app/lib/vacuum/media_attachments_vacuum.rb create mode 100644 app/lib/vacuum/preview_cards_vacuum.rb create mode 100644 app/lib/vacuum/statuses_vacuum.rb create mode 100644 app/lib/vacuum/system_keys_vacuum.rb (limited to 'app/lib') diff --git a/app/lib/redis_configuration.rb b/app/lib/redis_configuration.rb index e14d6c8b670..f0e86d985b8 100644 --- a/app/lib/redis_configuration.rb +++ b/app/lib/redis_configuration.rb @@ -7,9 +7,7 @@ class RedisConfiguration @pool = ConnectionPool.new(size: new_pool_size) { new.connection } end - def with - pool.with { |redis| yield redis } - end + delegate :with, to: :pool def pool @pool ||= establish_pool(pool_size) @@ -17,7 +15,7 @@ class RedisConfiguration def pool_size if Sidekiq.server? - Sidekiq.options[:concurrency] + Sidekiq[:concurrency] else ENV['MAX_THREADS'] || 5 end diff --git a/app/lib/vacuum.rb b/app/lib/vacuum.rb new file mode 100644 index 00000000000..9db1ec90bed --- /dev/null +++ b/app/lib/vacuum.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +module Vacuum; end diff --git a/app/lib/vacuum/access_tokens_vacuum.rb b/app/lib/vacuum/access_tokens_vacuum.rb new file mode 100644 index 00000000000..4f3878027ab --- /dev/null +++ b/app/lib/vacuum/access_tokens_vacuum.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class Vacuum::AccessTokensVacuum + def perform + vacuum_revoked_access_tokens! + vacuum_revoked_access_grants! + end + + private + + def vacuum_revoked_access_tokens! + Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all + end + + def vacuum_revoked_access_grants! + Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all + end +end diff --git a/app/lib/vacuum/backups_vacuum.rb b/app/lib/vacuum/backups_vacuum.rb new file mode 100644 index 00000000000..3b83072f369 --- /dev/null +++ b/app/lib/vacuum/backups_vacuum.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class Vacuum::BackupsVacuum + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_expired_backups! if retention_period? + end + + private + + def vacuum_expired_backups! + backups_past_retention_period.in_batches.destroy_all + end + + def backups_past_retention_period + Backup.unscoped.where(Backup.arel_table[:created_at].lt(@retention_period.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb new file mode 100644 index 00000000000..f46bcf75f7f --- /dev/null +++ b/app/lib/vacuum/feeds_vacuum.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class Vacuum::FeedsVacuum + def perform + vacuum_inactive_home_feeds! + vacuum_inactive_list_feeds! + end + + private + + def vacuum_inactive_home_feeds! + inactive_users.select(:id, :account_id).find_in_batches do |users| + feed_manager.clean_feeds!(:home, users.map(&:account_id)) + end + end + + def vacuum_inactive_list_feeds! + inactive_users_lists.select(:id).find_in_batches do |lists| + feed_manager.clean_feeds!(:list, lists.map(&:id)) + end + end + + def inactive_users + User.confirmed.inactive + end + + def inactive_users_lists + List.where(account_id: inactive_users.select(:account_id)) + end + + def feed_manager + FeedManager.instance + end +end diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb new file mode 100644 index 00000000000..7fb347ce4d5 --- /dev/null +++ b/app/lib/vacuum/media_attachments_vacuum.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class Vacuum::MediaAttachmentsVacuum + TTL = 1.day.freeze + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_cached_files! if retention_period? + vacuum_orphaned_records! + end + + private + + def vacuum_cached_files! + media_attachments_past_retention_period.find_each do |media_attachment| + media_attachment.file.destroy + media_attachment.thumbnail.destroy + media_attachment.save + end + end + + def vacuum_orphaned_records! + orphaned_media_attachments.in_batches.destroy_all + end + + def media_attachments_past_retention_period + MediaAttachment.unscoped.remote.cached.where(MediaAttachment.arel_table[:created_at].lt(@retention_period.ago)).where(MediaAttachment.arel_table[:updated_at].lt(@retention_period.ago)) + end + + def orphaned_media_attachments + MediaAttachment.unscoped.unattached.where(MediaAttachment.arel_table[:created_at].lt(TTL.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb new file mode 100644 index 00000000000..84ef100ed94 --- /dev/null +++ b/app/lib/vacuum/preview_cards_vacuum.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class Vacuum::PreviewCardsVacuum + TTL = 1.day.freeze + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_cached_images! if retention_period? + vacuum_orphaned_records! + end + + private + + def vacuum_cached_images! + preview_cards_past_retention_period.find_each do |preview_card| + preview_card.image.destroy + preview_card.save + end + end + + def vacuum_orphaned_records! + orphaned_preview_cards.in_batches.destroy_all + end + + def preview_cards_past_retention_period + PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago)) + end + + def orphaned_preview_cards + PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb new file mode 100644 index 00000000000..41d6ba270cc --- /dev/null +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class Vacuum::StatusesVacuum + include Redisable + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_statuses! if retention_period? + end + + private + + def vacuum_statuses! + statuses_scope.find_in_batches do |statuses| + # Side-effects not covered by foreign keys, such + # as the search index, must be handled first. + + remove_from_account_conversations(statuses) + remove_from_search_index(statuses) + + # Foreign keys take care of most associated records + # for us. Media attachments will be orphaned. + + Status.where(id: statuses.map(&:id)).delete_all + end + end + + def statuses_scope + Status.unscoped.kept.where(account: Account.remote).where(Status.arel_table[:id].lt(retention_period_as_id)).select(:id, :visibility) + end + + def retention_period_as_id + Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false) + end + + def analyze_statuses! + ActiveRecord::Base.connection.execute('ANALYZE statuses') + end + + def remove_from_account_conversations(statuses) + Status.where(id: statuses.select(&:direct_visibility?).map(&:id)).includes(:account, mentions: :account).each(&:unlink_from_conversations) + end + + def remove_from_search_index(statuses) + with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', statuses.map(&:id)) } if Chewy.enabled? + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/system_keys_vacuum.rb b/app/lib/vacuum/system_keys_vacuum.rb new file mode 100644 index 00000000000..ceee2fd164b --- /dev/null +++ b/app/lib/vacuum/system_keys_vacuum.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class Vacuum::SystemKeysVacuum + def perform + vacuum_expired_system_keys! + end + + private + + def vacuum_expired_system_keys! + SystemKey.expired.delete_all + end +end -- cgit v1.2.3