Retry on timeout from S3

[audax]

Traceback for the timeout that happened on a download from S3:

#012Traceback (most recent call last):
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/botocore/vendored/requests/packages/urllib3/response.py", line 243, in read
data = self._fp.read(amt)
File "/usr/lib/python3.5/http/client.py", line 433, in read
n = self.readinto(b)
File "/usr/lib/python3.5/http/client.py", line 473, in readinto
n = self.fp.readinto(b)
File "/usr/lib/python3.5/socket.py", line 575, in readinto
return self._sock.recv_into(b)
File "/usr/lib/python3.5/ssl.py", line 924, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib/python3.5/ssl.py", line 786, in read
return self._sslobj.read(len, buffer)
File "/usr/lib/python3.5/ssl.py", line 570, in read
v = self._sslobj.read(len, buffer)#012socket.timeout: The read operation timed out#012#012During handling of the above exception, another exception occurred:#012#012Traceback (most recent call last):
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/tornado/web.py", line 1445, in _execute
result = yield result
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/tornado/gen.py", line 1008, in run
value = future.result()
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/tornado/concurrent.py", line 232, in result
ra ise_exc_info(self._exc_info)
File "<string>", line 3, in raise_exc_info
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/tornado/gen.py", line 1014, in run
yielded = self.gen.throw(*exc_info)
File "/var/www/qabel-block/qabel-block/src/blockserver/server.py", line 161, in get
storage_object = yield self.retrieve_file(prefix, file_path, etag)
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/tornado/gen.py", line 1008, in run
value = future.result()
File "/usr/lib/python3.5/concurrent/futures/_base.py", line 398, in result
return self.__get_result()
File "/usr/lib/python3.5/concurrent/futures/_base.py", line 357, in __get_result
raise self._exception
File "/usr/lib/python3.5/concurrent/futures/thread.py", line 55, in run
result = self.fn(*self.args, **self.kwargs)
File "/var/www/qabel-block/qabel-block/src/blockserver/server.py", line 224, in retrieve_file
return self.transfer .retrieve(StorageObject(prefix, file_path, etag, None))
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/prometheus_client/core.py", line 665, in wrapped
return f(*args, **kwargs)
File "/var/www/qabel-block/qabel-block/src/blockserver/backend/transfer.py", line 131, in retrieve
for chunk in iter(lambda: streaming_body.read(8192), b''):
File "/var/www/qabel-block/qabel-block/src/blockserver/backend/transfer.py", line 131, in <lambda>
for chunk in iter(lambda: streaming_body.read(8192), b''):
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/botocore/response.py", line 74, in read
chunk = self._raw_stream.read(amt)
File "/var/www/qabel-block/venv/lib/python3.5/site-packages/botocore/vendored/requests/packages/urllib3/response.py", line 258, in read
raise ReadTimeoutError(self._pool, None, 'Read timed out.')#012botocore.vendored.requests.packages.urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(hos t='qabel.s3.amazonaws.com', port=443): Read timed out.

The error happens very rarely.

(edited by @enkore: inline-code-tag -> block-code-tag to make the trace readable)