From 265d00d5d6ffa12fb9d0b863adbe4c524758e04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Kr=C3=A1l?= Date: Mon, 24 Nov 2014 17:08:23 +0100 Subject: [PATCH] Retry on some HTTP statuses and touching up the retry logic --- docs/exceptions.rst | 4 ++++ docs/index.rst | 15 +++++++++++++++ elasticsearch/exceptions.py | 4 ++-- elasticsearch/transport.py | 24 +++++++++++++++--------- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/exceptions.rst b/docs/exceptions.rst index 8e6e1887..906bb7a5 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -14,6 +14,10 @@ Exceptions .. autoclass:: TransportError(ElasticsearchException) :members: +.. autoclass:: ConnectionError(TransportError) +.. autoclass:: ConnectionTimeout(ConnectionError) +.. autoclass:: SSLError(ConnectionError) + .. autoclass:: NotFoundError(TransportError) .. autoclass:: ConflictError(TransportError) .. autoclass:: RequestError(TransportError) diff --git a/docs/index.rst b/docs/index.rst index 0d0ca81f..a288137a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -89,6 +89,21 @@ of the relevant component and pass it in as a parameter to be used instead of the default implementation. +Automatic Retries +~~~~~~~~~~~~~~~~~ + +If a connection to a node fails due to connection issues (raises +:class:`~elasticsearch.ConnectionError`) it is considered in faulty state. It +will be placed on hold for ``dead_timeout`` seconds and the request will be +retried on another node. If a connection fails multiple times in a row the +timeout will get progressively larger to avoid hitting a node that's, by all +indication, down. If no live connection is availible, the connection that has +the smallest timeout will be used. + +By default retries are not triggered by a timeout +(:class:`~elasticsearch.ConnectionTimeout`), set ``retry_on_timeout`` to +``True`` to also retry on timeouts. + .. _sniffing: Sniffing diff --git a/elasticsearch/exceptions.py b/elasticsearch/exceptions.py index 2a98a0fa..e6e63fb1 100644 --- a/elasticsearch/exceptions.py +++ b/elasticsearch/exceptions.py @@ -63,11 +63,11 @@ class ConnectionError(TransportError): class SSLError(ConnectionError): - """ Error raised when encountering SSL errors. """ + """ Error raised when encountering SSL errors. """ class ConnectionTimeout(ConnectionError): - """ A network timeout. """ + """ A network timeout. Doesn't cause a node retry by default. """ def __str__(self): return 'ConnectionTimeout caused by - %s(%s)' % ( self.info.__class__.__name__, self.info) diff --git a/elasticsearch/transport.py b/elasticsearch/transport.py index e0b65a14..27e9b9ba 100644 --- a/elasticsearch/transport.py +++ b/elasticsearch/transport.py @@ -45,7 +45,7 @@ class Transport(object): connection_pool_class=ConnectionPool, host_info_callback=get_host_info, sniff_on_start=False, sniffer_timeout=None, sniff_timeout=.1, sniff_on_connection_fail=False, serializer=JSONSerializer(), serializers=None, - default_mimetype='application/json', max_retries=3, + default_mimetype='application/json', max_retries=3, retry_on_status=(503, 504, ), retry_on_timeout=False, send_get_body_as='GET', **kwargs): """ :arg hosts: list of dictionaries, each containing keyword arguments to @@ -68,6 +68,8 @@ class Transport(object): :arg default_mimetype: when no mimetype is specified by the server response assume this mimetype, defaults to `'application/json'` :arg max_retries: maximum number of retries before an exception is propagated + :arg retry_on_status: set of HTTP status codes on which we should retry + on a different node. defaults to ``(503, 504, )`` :arg retry_on_timeout: should timeout trigger a retry on different node? (default `False`) :arg send_get_body_as: for GET requests with body this option allows @@ -93,6 +95,7 @@ class Transport(object): self.max_retries = max_retries self.retry_on_timeout = retry_on_timeout + self.retry_on_status = retry_on_status self.send_get_body_as = send_get_body_as # data serializer @@ -292,8 +295,17 @@ class Transport(object): try: status, headers, data = connection.perform_request(method, url, params, body, ignore=ignore, timeout=timeout) - except ConnectionTimeout: - if self.retry_on_timeout: + + except TransportError as e: + retry = False + if isinstance(e, ConnectionTimeout): + retry = self.retry_on_timeout + elif isinstance(e, ConnectionError): + retry = True + elif e.status_code in self.retry_on_status: + retry = True + + if retry: # only mark as dead if we are retrying self.mark_dead(connection) # raise exception on last retry @@ -302,12 +314,6 @@ class Transport(object): else: raise - except ConnectionError: - self.mark_dead(connection) - - # raise exception on last retry - if attempt == self.max_retries: - raise else: # connection didn't fail, confirm it's live status self.connection_pool.mark_live(connection)