diff --git a/lib/algolia/error.rb b/lib/algolia/error.rb
index b7484f14..49bdad61 100644
--- a/lib/algolia/error.rb
+++ b/lib/algolia/error.rb
@@ -10,6 +10,15 @@ class AlgoliaError < StandardError
   # Used when hosts are unreachable
   #
   class AlgoliaUnreachableHostError < AlgoliaError
+    attr_reader :errors
+
+    def initialize(message, errors = [])
+      errors.last&.tap do |last_error|
+        message += " Last error for #{last_error[:host]}: #{last_error[:error]}"
+      end
+      super(message)
+      @errors = errors
+    end
   end
 
   # An exception class raised when the REST API returns an error.
diff --git a/lib/algolia/logger_helper.rb b/lib/algolia/logger_helper.rb
index 0500a378..7f490055 100644
--- a/lib/algolia/logger_helper.rb
+++ b/lib/algolia/logger_helper.rb
@@ -5,8 +5,17 @@ class LoggerHelper
     # @param debug_file [nil|String] file used to output the logs
     #
     def self.create(debug_file = nil)
-      file              = debug_file || (ENV['ALGOLIA_DEBUG'] ? File.new('debug.log') : $stderr)
-      instance          = ::Logger.new file
+      file = debug_file
+
+      if file.nil? && ENV['ALGOLIA_DEBUG']
+        begin
+          file = File.new('debug.log', 'a+')
+        rescue Errno::EACCES, Errno::ENOENT => e
+          puts "Failed to open debug.log: #{e.message}. Falling back to $stderr."
+        end
+      end
+
+      instance = ::Logger.new(file || $stderr)
       instance.progname = 'algolia'
       instance
     end
diff --git a/lib/algolia/transport/retry_strategy.rb b/lib/algolia/transport/retry_strategy.rb
index 48055995..57576c2a 100644
--- a/lib/algolia/transport/retry_strategy.rb
+++ b/lib/algolia/transport/retry_strategy.rb
@@ -1,6 +1,6 @@
 module Algolia
   module Transport
-    # Class RetryStatregy
+    # Class RetryStrategy
     class RetryStrategy
       include RetryOutcomeType
 
diff --git a/lib/algolia/transport/transport.rb b/lib/algolia/transport/transport.rb
index 80719bce..6a0a1693 100644
--- a/lib/algolia/transport/transport.rb
+++ b/lib/algolia/transport/transport.rb
@@ -49,6 +49,8 @@ def write(method, path, body = {}, opts = {})
       # @return [Response] response of the request
       #
       def request(call_type, method, path, body = {}, opts = {})
+        retry_errors = []
+
         @retry_strategy.get_tryable_hosts(call_type).each do |host|
           opts[:timeout]         ||= get_timeout(call_type) * (host.retry_count + 1)
           opts[:connect_timeout] ||= @config.connect_timeout * (host.retry_count + 1)
@@ -73,10 +75,14 @@ def request(call_type, method, path, body = {}, opts = {})
             decoded_error = json_to_hash(response.error, @config.symbolize_keys)
             raise AlgoliaHttpError.new(get_option(decoded_error, 'status'), get_option(decoded_error, 'message'))
           end
-          return json_to_hash(response.body, @config.symbolize_keys) unless outcome == RETRY
+          if outcome == RETRY
+            retry_errors << {host: host.url, error: response.error}
+          else
+            return json_to_hash(response.body, @config.symbolize_keys)
+          end
         end
 
-        raise AlgoliaUnreachableHostError, 'Unreachable hosts'
+        raise AlgoliaUnreachableHostError.new("Unreachable hosts.", retry_errors)
       end
 
       private
diff --git a/test/algolia/unit/retry_strategy_test.rb b/test/algolia/unit/retry_strategy_test.rb
index d9128cea..7da1360c 100644
--- a/test/algolia/unit/retry_strategy_test.rb
+++ b/test/algolia/unit/retry_strategy_test.rb
@@ -73,7 +73,7 @@ def test_resets_all_hosts_when_expired_according_to_write_type
 
   describe 'All hosts are unreachable' do
     def test_failure_when_all_hosts_are_down
-      stateful_hosts = ['0.0.0.0']
+      stateful_hosts = ['0.0.0.0', '1.0.0.0']
       @config        = Algolia::Search::Config.new(application_id: 'foo', api_key: 'bar', custom_hosts: stateful_hosts)
       client         = Algolia::Search::Client.create_with_config(@config)
       index          = client.init_index(get_test_index_name('failure'))
@@ -82,11 +82,15 @@ def test_failure_when_all_hosts_are_down
         index.save_object({ objectID: 'one' })
       end
 
-      assert_equal 'Unreachable hosts', exception.message
+      assert_includes exception.message, 'Unreachable hosts. Last error for 1.0.0.0: SSL_connect'
+      assert_equal exception.errors.size, 2
+      assert_equal exception.errors.last.keys, [:host, :error]
+      assert_equal exception.errors.first[:host], '0.0.0.0'
+      assert_equal exception.errors.last[:host], '1.0.0.0'
     end
   end
 
-  describe 'retry stategy decisions' do
+  describe 'retry strategy decisions' do
     def before_all
       super
       @app_id         = 'app_id'