From a2a7e0f949a3d7a7a7411d01129a32e866b0d822 Mon Sep 17 00:00:00 2001
From: Jonathan Hyman <jonathan.hyman@post.harvard.edu>
Date: Tue, 1 Sep 2015 14:23:19 -0400
Subject: [PATCH 1/2] Adds more resiliency to retrying read operations during
 failures.

---
 lib/mongo/retryable.rb       | 10 +++++++
 spec/mongo/retryable_spec.rb | 52 ++++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/lib/mongo/retryable.rb b/lib/mongo/retryable.rb
index 5e67b2d58d..c7ad0bcc08 100644
--- a/lib/mongo/retryable.rb
+++ b/lib/mongo/retryable.rb
@@ -24,6 +24,10 @@ module Retryable
     # @since 2.1.0
     NOT_MASTER = 'not master'.freeze
 
+    # Error codes received around reconfiguration
+    CONNECTION_ERRORS_RECONFIGURATION_CODES = [ 15988, 10276, 11600, 9001, 13639, 10009, 11002, 7 ].map(&:to_s).freeze
+    CONNECTION_ERRORS_MAGIC_STRINGS = [ 'could not get last error', 'connection attempt failed' ].freeze
+
     # Execute a read operation with a retry.
     #
     # @example Execute the read.
@@ -43,6 +47,12 @@ def read_with_retry(&block)
         block.call
       rescue Error::SocketError, Error::SocketTimeoutError
         retry_operation(&block)
+      rescue Error::OperationFailure => e
+        if CONNECTION_ERRORS_RECONFIGURATION_CODES.any? {|code| e.message.include?(code) } || CONNECTION_ERRORS_MAGIC_STRINGS.any? {|str| e.message.include?(str)}
+          retry_operation(&block)
+        else
+          raise e
+        end
       end
     end
 
diff --git a/spec/mongo/retryable_spec.rb b/spec/mongo/retryable_spec.rb
index 3a622510bc..f7fdedb321 100644
--- a/spec/mongo/retryable_spec.rb
+++ b/spec/mongo/retryable_spec.rb
@@ -66,6 +66,58 @@ def write
       end
     end
 
+    context 'when an operation error occurs that could not get last error' do
+
+      before do
+        expect(operation).to receive(:execute).and_raise(Mongo::Error::OperationFailure.new("could not get last error")).ordered
+        expect(cluster).to receive(:scan!).and_return(true).ordered
+        expect(operation).to receive(:execute).and_return(true).ordered
+      end
+
+      it 'executes the operation twice' do
+        expect(retryable.read).to be true
+      end
+    end
+
+    context 'when an operation error occurs that had a connection attempt failed' do
+
+      before do
+        expect(operation).to receive(:execute).and_raise(Mongo::Error::OperationFailure.new("connection attempt failed")).ordered
+        expect(cluster).to receive(:scan!).and_return(true).ordered
+        expect(operation).to receive(:execute).and_return(true).ordered
+      end
+
+      it 'executes the operation twice' do
+        expect(retryable.read).to be true
+      end
+    end
+
+    context 'when an operation error occurs with code 15988' do
+
+      before do
+        expect(operation).to receive(:execute).and_raise(Mongo::Error::OperationFailure.new("error querying server (15988)")).ordered
+        expect(cluster).to receive(:scan!).and_return(true).ordered
+        expect(operation).to receive(:execute).and_return(true).ordered
+      end
+
+      it 'executes the operation twice' do
+        expect(retryable.read).to be true
+      end
+    end
+
+    context 'when an operation error occurs with code 13639' do
+
+      before do
+        expect(operation).to receive(:execute).and_raise(Mongo::Error::OperationFailure.new("connection attempt failed (13639)")).ordered
+        expect(cluster).to receive(:scan!).and_return(true).ordered
+        expect(operation).to receive(:execute).and_return(true).ordered
+      end
+
+      it 'executes the operation twice' do
+        expect(retryable.read).to be true
+      end
+    end
+
     context 'when a socket timeout error occurs' do
 
       before do

From 86053b8634cccebb539a5dc9753627555c47491b Mon Sep 17 00:00:00 2001
From: Jonathan Hyman <jonathan.hyman@post.harvard.edu>
Date: Thu, 3 Sep 2015 14:53:11 -0400
Subject: [PATCH 2/2] Retry reads when deserializing a reply.

---
 lib/mongo/server/connectable.rb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/mongo/server/connectable.rb b/lib/mongo/server/connectable.rb
index ba716519c0..a334eefda9 100644
--- a/lib/mongo/server/connectable.rb
+++ b/lib/mongo/server/connectable.rb
@@ -20,6 +20,10 @@ class Server
     # @since 2.0.0
     module Connectable
 
+      def self.included(base)
+        base.__send__(:include, Mongo::Retryable)
+      end
+
       # The ssl option prefix.
       #
       # @since 2.1.0
@@ -103,7 +107,9 @@ def ensure_same_process!
       end
 
       def read
-        ensure_connected{ |socket| Protocol::Reply.deserialize(socket) }
+        read_with_retry do
+          ensure_connected{ |socket| Protocol::Reply.deserialize(socket) }
+        end
       end
     end
   end