Change algorithm of follow recommendations (#28314)

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

35 files changed, 297 insertions, 292 deletions

diff --git a/.github/renovate.json5 b/.github/renovate.json5
index aca52285940..dab99829a1b 100644
--- a/.github/renovate.json5
+++ b/.github/renovate.json5
@@ -102,9 +102,12 @@
     {
       // Group actions/*-artifact in the same PR
       matchManagers: ['github-actions'],
-      matchPackageNames: ['actions/download-artifact', 'actions/upload-artifact'],
+      matchPackageNames: [
+        'actions/download-artifact',
+        'actions/upload-artifact',
+      ],
       matchUpdateTypes: ['major'],
-      groupName: 'artifact actions (major)'
+      groupName: 'artifact actions (major)',
     },
     {
       // Update @types/* packages every week, with one grouped PR
diff --git a/app/controllers/admin/follow_recommendations_controller.rb b/app/controllers/admin/follow_recommendations_controller.rb
index 841e3cc7fbf..a54e41bd8c1 100644
--- a/app/controllers/admin/follow_recommendations_controller.rb
+++ b/app/controllers/admin/follow_recommendations_controller.rb
@@ -8,7 +8,7 @@ module Admin
       authorize :follow_recommendation, :show?
 
       @form     = Form::AccountBatch.new
-      @accounts = filtered_follow_recommendations
+      @accounts = filtered_follow_recommendations.page(params[:page])
     end
 
     def update
diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb
index 9737ae5cb62..9ba1cef63ca 100644
--- a/app/controllers/api/v1/suggestions_controller.rb
+++ b/app/controllers/api/v1/suggestions_controller.rb
@@ -3,22 +3,23 @@
 class Api::V1::SuggestionsController < Api::BaseController
   include Authorization
 
-  before_action -> { doorkeeper_authorize! :read }
+  before_action -> { doorkeeper_authorize! :read, :'read:accounts' }, only: :index
+  before_action -> { doorkeeper_authorize! :write, :'write:accounts' }, except: :index
   before_action :require_user!
+  before_action :set_suggestions
 
   def index
-    suggestions = suggestions_source.get(current_account, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT))
-    render json: suggestions.map(&:account), each_serializer: REST::AccountSerializer
+    render json: @suggestions.get(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:offset].to_i).map(&:account), each_serializer: REST::AccountSerializer
   end
 
   def destroy
-    suggestions_source.remove(current_account, params[:id])
+    @suggestions.remove(params[:id])
     render_empty
   end
 
   private
 
-  def suggestions_source
-    AccountSuggestions::PastInteractionsSource.new
+  def set_suggestions
+    @suggestions = AccountSuggestions.new(current_account)
   end
 end
diff --git a/app/controllers/api/v2/suggestions_controller.rb b/app/controllers/api/v2/suggestions_controller.rb
index 35eb276c01f..8516796e860 100644
--- a/app/controllers/api/v2/suggestions_controller.rb
+++ b/app/controllers/api/v2/suggestions_controller.rb
@@ -3,17 +3,23 @@
 class Api::V2::SuggestionsController < Api::BaseController
   include Authorization
 
-  before_action -> { doorkeeper_authorize! :read }
+  before_action -> { doorkeeper_authorize! :read, :'read:accounts' }, only: :index
+  before_action -> { doorkeeper_authorize! :write, :'write:accounts' }, except: :index
   before_action :require_user!
   before_action :set_suggestions
 
   def index
-    render json: @suggestions, each_serializer: REST::SuggestionSerializer
+    render json: @suggestions.get(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:offset].to_i), each_serializer: REST::SuggestionSerializer
+  end
+
+  def destroy
+    @suggestions.remove(params[:id])
+    render_empty
   end
 
   private
 
   def set_suggestions
-    @suggestions = AccountSuggestions.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT))
+    @suggestions = AccountSuggestions.new(current_account)
   end
 end
diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb
deleted file mode 100644
index f5bc2034659..00000000000
--- a/app/lib/potential_friendship_tracker.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-class PotentialFriendshipTracker
-  EXPIRE_AFTER = 90.days.seconds
-  MAX_ITEMS    = 80
-
-  WEIGHTS = {
-    reply: 1,
-    favourite: 10,
-    reblog: 20,
-  }.freeze
-
-  class << self
-    include Redisable
-
-    def record(account_id, target_account_id, action)
-      return if account_id == target_account_id
-
-      key    = "interactions:#{account_id}"
-      weight = WEIGHTS[action]
-
-      redis.zincrby(key, weight, target_account_id)
-      redis.zremrangebyrank(key, 0, -MAX_ITEMS)
-      redis.expire(key, EXPIRE_AFTER)
-    end
-
-    def remove(account_id, target_account_id)
-      redis.zrem("interactions:#{account_id}", target_account_id)
-    end
-  end
-end
diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb
index db2e37184f1..753935d6af6 100644
--- a/app/models/account_domain_block.rb
+++ b/app/models/account_domain_block.rb
@@ -19,6 +19,7 @@ class AccountDomainBlock < ApplicationRecord
   validates :domain, presence: true, uniqueness: { scope: :account_id }, domain: true
 
   after_commit :invalidate_domain_blocking_cache
+  after_commit :invalidate_follow_recommendations_cache
 
   private
 
@@ -26,4 +27,8 @@ class AccountDomainBlock < ApplicationRecord
     Rails.cache.delete("exclude_domains_for:#{account_id}")
     Rails.cache.delete(['exclude_domains', account_id, domain])
   end
+
+  def invalidate_follow_recommendations_cache
+    Rails.cache.delete("follow_recommendations/#{account_id}")
+  end
 end
diff --git a/app/models/account_suggestions.rb b/app/models/account_suggestions.rb
index d1774e62fae..d62176c7ca3 100644
--- a/app/models/account_suggestions.rb
+++ b/app/models/account_suggestions.rb
@@ -1,28 +1,48 @@
 # frozen_string_literal: true
 
 class AccountSuggestions
+  include DatabaseHelper
+
   SOURCES = [
     AccountSuggestions::SettingSource,
-    AccountSuggestions::PastInteractionsSource,
+    AccountSuggestions::FriendsOfFriendsSource,
+    AccountSuggestions::SimilarProfilesSource,
     AccountSuggestions::GlobalSource,
   ].freeze
 
-  def self.get(account, limit)
-    SOURCES.each_with_object([]) do |source_class, suggestions|
-      source_suggestions = source_class.new.get(
-        account,
-        skip_account_ids: suggestions.map(&:account_id),
-        limit: limit - suggestions.size
-      )
+  BATCH_SIZE = 40
 
-      suggestions.concat(source_suggestions)
-    end
+  def initialize(account)
+    @account = account
   end
 
-  def self.remove(account, target_account_id)
-    SOURCES.each do |source_class|
-      source = source_class.new
-      source.remove(account, target_account_id)
+  def get(limit, offset = 0)
+    with_read_replica do
+      account_ids_with_sources = Rails.cache.fetch("follow_recommendations/#{@account.id}", expires_in: 15.minutes) do
+        SOURCES.flat_map { |klass| klass.new.get(@account, limit: BATCH_SIZE) }.each_with_object({}) do |(account_id, source), h|
+          (h[account_id] ||= []).concat(Array(source).map(&:to_sym))
+        end.to_a.shuffle
+      end
+
+      # The sources deliver accounts that haven't yet been followed, are not blocked,
+      # and so on. Since we reset the cache on follows, blocks, and so on, we don't need
+      # a complicated query on this end.
+
+      account_ids  = account_ids_with_sources[offset, limit]
+      accounts_map = Account.where(id: account_ids.map(&:first)).includes(:account_stat).index_by(&:id)
+
+      account_ids.filter_map do |(account_id, source)|
+        next unless accounts_map.key?(account_id)
+
+        AccountSuggestions::Suggestion.new(
+          account: accounts_map[account_id],
+          source: source
+        )
+      end
     end
   end
+
+  def remove(target_account_id)
+    FollowRecommendationMute.create(account_id: @account.id, target_account_id: target_account_id)
+  end
 end
diff --git a/app/models/account_suggestions/friends_of_friends_source.rb b/app/models/account_suggestions/friends_of_friends_source.rb
new file mode 100644
index 00000000000..77d4f635aa6
--- /dev/null
+++ b/app/models/account_suggestions/friends_of_friends_source.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::FriendsOfFriendsSource < AccountSuggestions::Source
+  def get(account, limit: 10)
+    Account.find_by_sql([<<~SQL.squish, { id: account.id, limit: limit }]).map { |row| [row.id, key] }
+      WITH first_degree AS (
+          SELECT target_account_id
+          FROM follows
+          JOIN accounts AS target_accounts ON follows.target_account_id = target_accounts.id
+          WHERE account_id = :id
+            AND NOT target_accounts.hide_collections
+      )
+      SELECT accounts.id, COUNT(*) AS frequency
+      FROM accounts
+      JOIN follows ON follows.target_account_id = accounts.id
+      JOIN account_stats ON account_stats.account_id = accounts.id
+      LEFT OUTER JOIN follow_recommendation_mutes ON follow_recommendation_mutes.target_account_id = accounts.id AND follow_recommendation_mutes.account_id = :id
+      WHERE follows.account_id IN (SELECT * FROM first_degree)
+        AND follows.target_account_id NOT IN (SELECT * FROM first_degree)
+        AND follows.target_account_id <> :id
+        AND accounts.discoverable
+        AND accounts.suspended_at IS NULL
+        AND accounts.silenced_at IS NULL
+        AND accounts.moved_to_account_id IS NULL
+        AND follow_recommendation_mutes.target_account_id IS NULL
+      GROUP BY accounts.id, account_stats.id
+      ORDER BY frequency DESC, account_stats.followers_count ASC
+      LIMIT :limit
+    SQL
+  end
+
+  private
+
+  def key
+    :friends_of_friends
+  end
+end
diff --git a/app/models/account_suggestions/global_source.rb b/app/models/account_suggestions/global_source.rb
index 651041d6751..d68f285e4f4 100644
--- a/app/models/account_suggestions/global_source.rb
+++ b/app/models/account_suggestions/global_source.rb
@@ -1,39 +1,13 @@
 # frozen_string_literal: true
 
 class AccountSuggestions::GlobalSource < AccountSuggestions::Source
-  include Redisable
-
-  def key
-    :global
-  end
-
-  def get(account, skip_account_ids: [], limit: 40)
-    account_ids = account_ids_for_locale(I18n.locale.to_s.split(/[_-]/).first) - [account.id] - skip_account_ids
-
-    as_ordered_suggestions(
-      scope(account).where(id: account_ids),
-      account_ids
-    ).take(limit)
-  end
-
-  def remove(_account, _target_account_id)
-    nil
+  def get(account, limit: 10)
+    FollowRecommendation.localized(content_locale).joins(:account).merge(base_account_scope(account)).order(rank: :desc).limit(limit).pluck(:account_id, :reason)
   end
 
   private
 
-  def scope(account)
-    Account.searchable
-           .followable_by(account)
-           .not_excluded_by_account(account)
-           .not_domain_blocked_by_account(account)
-  end
-
-  def account_ids_for_locale(locale)
-    redis.zrevrange("follow_recommendations:#{locale}", 0, -1).map(&:to_i)
-  end
-
-  def to_ordered_list_key(account)
-    account.id
+  def content_locale
+    I18n.locale.to_s.split(/[_-]/).first
   end
 end
diff --git a/app/models/account_suggestions/past_interactions_source.rb b/app/models/account_suggestions/past_interactions_source.rb
deleted file mode 100644
index d169394f11a..00000000000
--- a/app/models/account_suggestions/past_interactions_source.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-class AccountSuggestions::PastInteractionsSource < AccountSuggestions::Source
-  include Redisable
-
-  def key
-    :past_interactions
-  end
-
-  def get(account, skip_account_ids: [], limit: 40)
-    account_ids = account_ids_for_account(account.id, limit + skip_account_ids.size) - skip_account_ids
-
-    as_ordered_suggestions(
-      scope.where(id: account_ids),
-      account_ids
-    ).take(limit)
-  end
-
-  def remove(account, target_account_id)
-    redis.zrem("interactions:#{account.id}", target_account_id)
-  end
-
-  private
-
-  def scope
-    Account.searchable
-  end
-
-  def account_ids_for_account(account_id, limit)
-    redis.zrevrange("interactions:#{account_id}", 0, limit).map(&:to_i)
-  end
-
-  def to_ordered_list_key(account)
-    account.id
-  end
-end
diff --git a/app/models/account_suggestions/setting_source.rb b/app/models/account_suggestions/setting_source.rb
index 6185732b4bc..4b7275bf7ad 100644
--- a/app/models/account_suggestions/setting_source.rb
+++ b/app/models/account_suggestions/setting_source.rb
@@ -1,32 +1,18 @@
 # frozen_string_literal: true
 
 class AccountSuggestions::SettingSource < AccountSuggestions::Source
-  def key
-    :staff
-  end
-
-  def get(account, skip_account_ids: [], limit: 40)
-    return [] unless setting_enabled?
-
-    as_ordered_suggestions(
-      scope(account).where(setting_to_where_condition).where.not(id: skip_account_ids),
-      usernames_and_domains
-    ).take(limit)
-  end
-
-  def remove(_account, _target_account_id)
-    nil
+  def get(account, limit: 10)
+    if setting_enabled?
+      base_account_scope(account).where(setting_to_where_condition).limit(limit).pluck(:id).zip([key].cycle)
+    else
+      []
+    end
   end
 
   private
 
-  def scope(account)
-    Account.searchable
-           .followable_by(account)
-           .not_excluded_by_account(account)
-           .not_domain_blocked_by_account(account)
-           .where(locked: false)
-           .where.not(id: account.id)
+  def key
+    :featured
   end
 
   def usernames_and_domains
@@ -61,8 +47,4 @@ class AccountSuggestions::SettingSource < AccountSuggestions::Source
   def setting
     Setting.bootstrap_timeline_accounts
   end
-
-  def to_ordered_list_key(account)
-    [account.username.downcase, account.domain&.downcase]
-  end
 end
diff --git a/app/models/account_suggestions/similar_profiles_source.rb b/app/models/account_suggestions/similar_profiles_source.rb
new file mode 100644
index 00000000000..733c5f0bbcd
--- /dev/null
+++ b/app/models/account_suggestions/similar_profiles_source.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::SimilarProfilesSource < AccountSuggestions::Source
+  class QueryBuilder < AccountSearchService::QueryBuilder
+    def must_clauses
+      [
+        {
+          more_like_this: {
+            fields: %w(text text.stemmed),
+            like: @query.map { |id| { _index: 'accounts', _id: id } },
+          },
+        },
+
+        {
+          term: {
+            properties: 'discoverable',
+          },
+        },
+      ]
+    end
+
+    def must_not_clauses
+      [
+        {
+          terms: {
+            id: following_ids,
+          },
+        },
+
+        {
+          term: {
+            properties: 'bot',
+          },
+        },
+      ]
+    end
+
+    def should_clauses
+      {
+        term: {
+          properties: {
+            value: 'verified',
+            boost: 2,
+          },
+        },
+      }
+    end
+  end
+
+  def get(account, limit: 10)
+    recently_followed_account_ids = account.active_relationships.recent.limit(5).pluck(:target_account_id)
+
+    if Chewy.enabled? && !recently_followed_account_ids.empty?
+      QueryBuilder.new(recently_followed_account_ids, account).build.limit(limit).hits.pluck('_id').map(&:to_i).zip([key].cycle)
+    else
+      []
+    end
+  rescue Faraday::ConnectionFailed
+    []
+  end
+
+  private
+
+  def key
+    :similar_to_recently_followed
+  end
+end
diff --git a/app/models/account_suggestions/source.rb b/app/models/account_suggestions/source.rb
index 504d26a8bd6..ee93a1342fc 100644
--- a/app/models/account_suggestions/source.rb
+++ b/app/models/account_suggestions/source.rb
@@ -1,34 +1,18 @@
 # frozen_string_literal: true
 
 class AccountSuggestions::Source
-  def key
-    raise NotImplementedError
-  end
-
   def get(_account, **kwargs)
     raise NotImplementedError
   end
 
-  def remove(_account, target_account_id)
-    raise NotImplementedError
-  end
-
   protected
 
-  def as_ordered_suggestions(scope, ordered_list)
-    return [] if ordered_list.empty?
-
-    map = scope.index_by { |account| to_ordered_list_key(account) }
-
-    ordered_list.filter_map { |ordered_list_key| map[ordered_list_key] }.map do |account|
-      AccountSuggestions::Suggestion.new(
-        account: account,
-        source: key
-      )
-    end
-  end
-
-  def to_ordered_list_key(_account)
-    raise NotImplementedError
+  def base_account_scope(account)
+    Account.searchable
+           .followable_by(account)
+           .not_excluded_by_account(account)
+           .not_domain_blocked_by_account(account)
+           .where.not(id: account.id)
+           .joins("LEFT OUTER JOIN follow_recommendation_mutes ON follow_recommendation_mutes.target_account_id = accounts.id AND follow_recommendation_mutes.account_id = #{account.id}").where(follow_recommendation_mutes: { target_account_id: nil })
   end
 end
diff --git a/app/models/block.rb b/app/models/block.rb
index 11156ebab31..5476542a5ab 100644
--- a/app/models/block.rb
+++ b/app/models/block.rb
@@ -26,15 +26,20 @@ class Block < ApplicationRecord
   end
 
   before_validation :set_uri, only: :create
-  after_commit :remove_blocking_cache
+  after_commit :invalidate_blocking_cache
+  after_commit :invalidate_follow_recommendations_cache
 
   private
 
-  def remove_blocking_cache
+  def invalidate_blocking_cache
     Rails.cache.delete("exclude_account_ids_for:#{account_id}")
     Rails.cache.delete("exclude_account_ids_for:#{target_account_id}")
   end
 
+  def invalidate_follow_recommendations_cache
+    Rails.cache.delete("follow_recommendations/#{account_id}")
+  end
+
   def set_uri
     self.uri = ActivityPub::TagManager.instance.generate_uri_for(self) if uri.nil?
   end
diff --git a/app/models/concerns/account/associations.rb b/app/models/concerns/account/associations.rb
index 31902ae21a8..2bb6fed5ad0 100644
--- a/app/models/concerns/account/associations.rb
+