Batched remove status service (#3735)

* Make Pubsubhubbub::DistributionWorker handle both single stream entry
arguments, as well as arrays of stream entries

* Add BatchedRemoveStatusService, make SuspendAccountService use it

* Improve method names

* Add test

* Add more tests

* Use PuSH payloads of 100 to have a clear mapping of
1000 input statuses -> 10 PuSH payloads

It was nice while it lasted

1 files changed, 31 insertions, 11 deletions

diff --git a/app/workers/pubsubhubbub/distribution_worker.rb b/app/workers/pubsubhubbub/distribution_worker.rb
index b8f5c35e16f..da01dcd91ea 100644
--- a/app/workers/pubsubhubbub/distribution_worker.rb
+++ b/app/workers/pubsubhubbub/distribution_worker.rb
@@ -5,24 +5,44 @@ class Pubsubhubbub::DistributionWorker
 
   sidekiq_options queue: 'push'
 
-  def perform(stream_entry_id)
-    stream_entry = StreamEntry.find(stream_entry_id)
+  def perform(stream_entry_ids)
+    stream_entries = StreamEntry.where(id: stream_entry_ids).includes(:status).reject { |e| e.status&.direct_visibility? }
 
-    return if stream_entry.status&.direct_visibility?
+    return if stream_entries.empty?
 
-    @account = stream_entry.account
-    @payload = AtomSerializer.render(AtomSerializer.new.feed(@account, [stream_entry]))
+    @account       = stream_entries.first.account
+    @subscriptions = active_subscriptions.to_a
+
+    distribute_public!(stream_entries.reject(&:hidden?))
+    distribute_hidden!(stream_entries.reject { |s| !s.hidden? })
+  end
+
+  private
+
+  def distribute_public!(stream_entries)
+    return if stream_entries.empty?
+
+    @payload = AtomSerializer.render(AtomSerializer.new.feed(@account, stream_entries))
+
+    Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription|
+      [subscription.id, @payload]
+    end
+  end
+
+  def distribute_hidden!(stream_entries)
+    return if stream_entries.empty?
+
+    @payload = AtomSerializer.render(AtomSerializer.new.feed(@account, stream_entries))
     @domains = @account.followers_domains
 
-    Subscription.where(account: @account).active.select('id, callback_url').find_each do |subscription|
-      next if stream_entry.hidden? && !allowed_to_receive?(subscription.callback_url)
-      Pubsubhubbub::DeliveryWorker.perform_async(subscription.id, @payload)
+    Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions.reject { |s| !allowed_to_receive?(s.callback_url) }) do |subscription|
+      [subscription.id, @payload]
     end
-  rescue ActiveRecord::RecordNotFound
-    true
   end
 
-  private
+  def active_subscriptions
+    Subscription.where(account: @account).active.select('id, callback_url')
+  end
 
   def allowed_to_receive?(callback_url)
     @domains.include?(Addressable::URI.parse(callback_url).host)