Add request pool to improve delivery performance (#10353)

* Add request pool to improve delivery performance

Fix #7909

* Ensure connection is closed when exception interrupts execution

* Remove Timeout#timeout from socket connection

* Fix infinite retrial loop on HTTP::ConnectionError

* Close sockets on failure, reduce idle time to 90 seconds

* Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server

* Use a shared pool size, 512 by default, to stay below open file limit

* Add some tests

* Add more tests

* Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds

* Use a shared pool that returns preferred connection but re-purposes other ones when needed

* Fix wrong connection being returned on subsequent calls within the same thread

* Reduce mutex calls on flushes from 2 to 1 and add test for reaping

3 files changed, 152 insertions, 0 deletions

diff --git a/spec/lib/connection_pool/shared_connection_pool_spec.rb b/spec/lib/connection_pool/shared_connection_pool_spec.rb
new file mode 100644
index 00000000000..1144645580b
--- /dev/null
+++ b/spec/lib/connection_pool/shared_connection_pool_spec.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe ConnectionPool::SharedConnectionPool do
+  class MiniConnection
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+    end
+  end
+
+  subject { described_class.new(size: 5, timeout: 5) { |site| MiniConnection.new(site) } }
+
+  describe '#with' do
+    it 'runs a block with a connection' do
+      block_run = false
+
+      subject.with('foo') do |connection|
+        expect(connection).to be_a MiniConnection
+        block_run = true
+      end
+
+      expect(block_run).to be true
+    end
+  end
+end
diff --git a/spec/lib/connection_pool/shared_timed_stack_spec.rb b/spec/lib/connection_pool/shared_timed_stack_spec.rb
new file mode 100644
index 00000000000..f680c596670
--- /dev/null
+++ b/spec/lib/connection_pool/shared_timed_stack_spec.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe ConnectionPool::SharedTimedStack do
+  class MiniConnection
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+    end
+  end
+
+  subject { described_class.new(5) { |site| MiniConnection.new(site) } }
+
+  describe '#push' do
+    it 'keeps the connection in the stack' do
+      subject.push(MiniConnection.new('foo'))
+      expect(subject.size).to eq 1
+    end
+  end
+
+  describe '#pop' do
+    it 'returns a connection' do
+      expect(subject.pop('foo')).to be_a MiniConnection
+    end
+
+    it 'returns the same connection that was pushed in' do
+      connection = MiniConnection.new('foo')
+      subject.push(connection)
+      expect(subject.pop('foo')).to be connection
+    end
+
+    it 'does not create more than maximum amount of connections' do
+      expect { 6.times { subject.pop('foo', 0) } }.to raise_error Timeout::Error
+    end
+
+    it 'repurposes a connection for a different site when maximum amount is reached' do
+      5.times { subject.push(MiniConnection.new('foo')) }
+      expect(subject.pop('bar')).to be_a MiniConnection
+    end
+  end
+
+  describe '#empty?' do
+    it 'returns true when no connections on the stack' do
+      expect(subject.empty?).to be true
+    end
+
+    it 'returns false when there are connections on the stack' do
+      subject.push(MiniConnection.new('foo'))
+      expect(subject.empty?).to be false
+    end
+  end
+
+  describe '#size' do
+    it 'returns the number of connections on the stack' do
+      2.times { subject.push(MiniConnection.new('foo')) }
+      expect(subject.size).to eq 2
+    end
+  end
+end
diff --git a/spec/lib/request_pool_spec.rb b/spec/lib/request_pool_spec.rb
new file mode 100644
index 00000000000..4a144d7c7f7
--- /dev/null
+++ b/spec/lib/request_pool_spec.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe RequestPool do
+  subject { described_class.new }
+
+  describe '#with' do
+    it 'returns a HTTP client for a host' do
+      subject.with('http://example.com') do |http_client|
+        expect(http_client).to be_a HTTP::Client
+      end
+    end
+
+    it 'returns the same instance of HTTP client within the same thread for the same host' do
+      test_client = nil
+
+      subject.with('http://example.com') { |http_client| test_client = http_client }
+      expect(test_client).to_not be_nil
+      subject.with('http://example.com') { |http_client| expect(http_client).to be test_client }
+    end
+
+    it 'returns different HTTP clients for different hosts' do
+      test_client = nil
+
+      subject.with('http://example.com') { |http_client| test_client = http_client }
+      expect(test_client).to_not be_nil
+      subject.with('http://example.org') { |http_client| expect(http_client).to_not be test_client }
+    end
+
+    it 'grows to the number of threads accessing it' do
+      stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!')
+
+      subject
+
+      threads = 20.times.map do |i|
+        Thread.new do
+          20.times do
+            subject.with('http://example.com') do |http_client|
+              http_client.get('/').flush
+            end
+          end
+        end
+      end
+
+      threads.map(&:join)
+
+      expect(subject.size).to be > 1
+    end
+
+    it 'closes idle connections' do
+      stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!')
+
+      subject.with('http://example.com') do |http_client|
+        http_client.get('/').flush
+      end
+
+      expect(subject.size).to eq 1
+      sleep RequestPool::MAX_IDLE_TIME + 30 + 1
+      expect(subject.size).to eq 0
+    end
+  end
+end