diff --git a/elasticsearch/client/__init__.py b/elasticsearch/client/__init__.py index 53ec0529..2673e6c1 100644 --- a/elasticsearch/client/__init__.py +++ b/elasticsearch/client/__init__.py @@ -264,7 +264,7 @@ class Elasticsearch(object): if len(cons) > 5: cons = cons[:5] + ["..."] return "<{cls}({cons})>".format(cls=self.__class__.__name__, cons=cons) - except: + except Exception: # probably operating on custom transport and connection_pool, ignore return super(Elasticsearch, self).__repr__() diff --git a/elasticsearch/client/utils.py b/elasticsearch/client/utils.py index cb7a9d94..8cd22159 100644 --- a/elasticsearch/client/utils.py +++ b/elasticsearch/client/utils.py @@ -33,7 +33,7 @@ def _escape(value): # encode strings to utf-8 if isinstance(value, string_types): - if PY2 and isinstance(value, unicode): + if PY2 and isinstance(value, unicode): # noqa: F821 return value.encode("utf-8") if not PY2 and isinstance(value, str): return value.encode("utf-8") diff --git a/elasticsearch/compat.py b/elasticsearch/compat.py index 70bcac16..aba63ea7 100644 --- a/elasticsearch/compat.py +++ b/elasticsearch/compat.py @@ -3,7 +3,7 @@ import sys PY2 = sys.version_info[0] == 2 if PY2: - string_types = (basestring,) + string_types = (basestring,) # noqa: F821 from urllib import quote_plus, quote, urlencode, unquote from urlparse import urlparse from itertools import imap as map @@ -14,3 +14,14 @@ else: map = map from queue import Queue + +__all__ = [ + "string_types", + "quote_plus", + "quote", + "urlencode", + "unquote", + "urlparse", + "map", + "Queue", +] diff --git a/elasticsearch/connection/base.py b/elasticsearch/connection/base.py index e908fae5..37386bec 100644 --- a/elasticsearch/connection/base.py +++ b/elasticsearch/connection/base.py @@ -57,7 +57,9 @@ class Connection(object): try: _, cloud_id = cloud_id.split(":") parent_dn, es_uuid, _ = ( - binascii.a2b_base64(cloud_id.encode("utf-8")).decode("utf-8").split("$") + binascii.a2b_base64(cloud_id.encode("utf-8")) + .decode("utf-8") + .split("$") ) except ValueError: raise ImproperlyConfigured("'cloud_id' is not properly formatted") @@ -106,9 +108,7 @@ class Connection(object): def __eq__(self, other): if not isinstance(other, Connection): - raise TypeError( - "Unsupported equality check for %s and %s" % (self, other) - ) + raise TypeError("Unsupported equality check for %s and %s" % (self, other)) return self.__hash__() == other.__hash__() def __hash__(self): @@ -242,6 +242,6 @@ class Connection(object): :arg api_key, either a tuple or a base64 encoded string """ if isinstance(api_key, (tuple, list)): - s = "{0}:{1}".format(api_key[0], api_key[1]).encode('utf-8') - return "ApiKey " + binascii.b2a_base64(s).rstrip(b"\r\n").decode('utf-8') + s = "{0}:{1}".format(api_key[0], api_key[1]).encode("utf-8") + return "ApiKey " + binascii.b2a_base64(s).rstrip(b"\r\n").decode("utf-8") return "ApiKey " + api_key diff --git a/elasticsearch/connection/http_urllib3.py b/elasticsearch/connection/http_urllib3.py index 936d5d87..8273df33 100644 --- a/elasticsearch/connection/http_urllib3.py +++ b/elasticsearch/connection/http_urllib3.py @@ -5,6 +5,15 @@ from urllib3.exceptions import ReadTimeoutError, SSLError as UrllibSSLError from urllib3.util.retry import Retry import warnings +from .base import Connection +from ..exceptions import ( + ConnectionError, + ImproperlyConfigured, + ConnectionTimeout, + SSLError, +) +from ..compat import urlencode + # sentinel value for `verify_certs` and `ssl_show_warn`. # This is used to detect if a user is passing in a value # for SSL kwargs if also using an SSLContext. @@ -20,15 +29,6 @@ try: except ImportError: pass -from .base import Connection -from ..exceptions import ( - ConnectionError, - ImproperlyConfigured, - ConnectionTimeout, - SSLError, -) -from ..compat import urlencode - def create_ssl_context(**kwargs): """ @@ -188,11 +188,7 @@ class Urllib3HttpConnection(Connection): urllib3.disable_warnings() self.pool = pool_class( - self.hostname, - port=self.port, - timeout=self.timeout, - maxsize=maxsize, - **kw + self.hostname, port=self.port, timeout=self.timeout, maxsize=maxsize, **kw ) def perform_request( diff --git a/elasticsearch/connection_pool.py b/elasticsearch/connection_pool.py index d9bdcd29..6881707c 100644 --- a/elasticsearch/connection_pool.py +++ b/elasticsearch/connection_pool.py @@ -150,7 +150,10 @@ class ConnectionPool(object): try: self.connections.remove(connection) except ValueError: - logger.info("Attempted to remove %r, but it does not exist in the connection pool.", connection) + logger.info( + "Attempted to remove %r, but it does not exist in the connection pool.", + connection, + ) # connection not alive or another thread marked it already, ignore return else: diff --git a/elasticsearch/helpers/__init__.py b/elasticsearch/helpers/__init__.py index 848138bf..28a11c30 100644 --- a/elasticsearch/helpers/__init__.py +++ b/elasticsearch/helpers/__init__.py @@ -2,3 +2,16 @@ from .errors import BulkIndexError, ScanError from .actions import expand_action, streaming_bulk, bulk, parallel_bulk from .actions import scan, reindex from .actions import _chunk_actions, _process_bulk_chunk + +__all__ = [ + "BulkIndexError", + "ScanError", + "expand_action", + "streaming_bulk", + "bulk", + "parallel_bulk", + "scan", + "reindex", + "_chunk_actions", + "_process_bulk_chunk", +] diff --git a/elasticsearch/helpers/actions.py b/elasticsearch/helpers/actions.py index 4e3cea1e..af409c37 100644 --- a/elasticsearch/helpers/actions.py +++ b/elasticsearch/helpers/actions.py @@ -45,7 +45,13 @@ def expand_action(data): "version_type", ): if key in data: - if key in ["_parent", "_retry_on_conflict", "_routing", "_version", "_version_type"]: + if key in [ + "_parent", + "_retry_on_conflict", + "_routing", + "_version", + "_version_type", + ]: action[op_type][key[1:]] = data.pop(key) else: action[op_type][key] = data.pop(key) diff --git a/elasticsearch/helpers/test.py b/elasticsearch/helpers/test.py index ebf11c25..0878969e 100644 --- a/elasticsearch/helpers/test.py +++ b/elasticsearch/helpers/test.py @@ -12,7 +12,10 @@ def get_test_client(nowait=False, **kwargs): if "PYTHON_CONNECTION_CLASS" in os.environ: from elasticsearch import connection - kw["connection_class"] = getattr(connection, os.environ["PYTHON_CONNECTION_CLASS"]) + + kw["connection_class"] = getattr( + connection, os.environ["PYTHON_CONNECTION_CLASS"] + ) kw.update(kwargs) client = Elasticsearch([os.environ.get("ELASTICSEARCH_HOST", {})], **kw) diff --git a/elasticsearch/transport.py b/elasticsearch/transport.py index 02fe79bf..2a2bde98 100644 --- a/elasticsearch/transport.py +++ b/elasticsearch/transport.py @@ -1,11 +1,9 @@ import time from itertools import chain -from platform import python_version from .connection import Urllib3HttpConnection from .connection_pool import ConnectionPool, DummyConnectionPool from .serializer import JSONSerializer, Deserializer, DEFAULT_SERIALIZERS -from . import __versionstr__ from .exceptions import ( ConnectionError, TransportError, @@ -230,7 +228,7 @@ class Transport(object): pass else: raise TransportError("N/A", "Unable to sniff hosts.") - except: + except Exception: # keep the previous value on error self.last_sniff = previous_sniff raise @@ -245,11 +243,11 @@ class Transport(object): if not address or ":" not in address: return None - if '/' in address: + if "/" in address: # Support 7.x host/ip:port behavior where http.publish_host has been set. - fqdn, ipaddress = address.split('/', 1) + fqdn, ipaddress = address.split("/", 1) host["host"] = fqdn - _, host["port"] = ipaddress.rsplit(':', 1) + _, host["port"] = ipaddress.rsplit(":", 1) host["port"] = int(host["port"]) else: diff --git a/test_elasticsearch/test_cases.py b/test_elasticsearch/test_cases.py index 6dec2b3c..7f71f358 100644 --- a/test_elasticsearch/test_cases.py +++ b/test_elasticsearch/test_cases.py @@ -1,11 +1,6 @@ from collections import defaultdict - -try: - # python 2.6 - from unittest2 import TestCase, SkipTest -except ImportError: - from unittest import TestCase, SkipTest - +from unittest import TestCase +from unittest import SkipTest # noqa: F401 from elasticsearch import Elasticsearch diff --git a/test_elasticsearch/test_connection.py b/test_elasticsearch/test_connection.py index fa1a94eb..e4aa8e13 100644 --- a/test_elasticsearch/test_connection.py +++ b/test_elasticsearch/test_connection.py @@ -25,9 +25,7 @@ def gzip_decompress(data): class TestUrllib3Connection(TestCase): - def _get_mock_connection( - self, connection_params={}, response_body=b"{}" - ): + def _get_mock_connection(self, connection_params={}, response_body=b"{}"): con = Urllib3HttpConnection(**connection_params) def _dummy_urlopen(*args, **kwargs): @@ -66,7 +64,9 @@ class TestUrllib3Connection(TestCase): con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" ) self.assertEquals(con.port, 9243) - self.assertEquals(con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com") + self.assertEquals( + con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com" + ) self.assertTrue(con.http_compress) def test_api_key_auth(self): @@ -75,16 +75,24 @@ class TestUrllib3Connection(TestCase): cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", api_key=("elastic", "changeme1"), ) - self.assertEquals(con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=") - self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243") + self.assertEquals( + con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=" + ) + self.assertEquals( + con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + ) # test with base64 encoded string con = Urllib3HttpConnection( cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=", ) - self.assertEquals(con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=") - self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243") + self.assertEquals( + con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=" + ) + self.assertEquals( + con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + ) def test_no_http_compression(self): con = self._get_mock_connection() @@ -134,19 +142,22 @@ class TestUrllib3Connection(TestCase): con = Urllib3HttpConnection( cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", - http_compress=False + http_compress=False, ) self.assertEquals(con.http_compress, False) con = Urllib3HttpConnection( cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", - http_compress=True + http_compress=True, ) self.assertEquals(con.http_compress, True) def test_default_user_agent(self): con = Urllib3HttpConnection() - self.assertEquals(con._get_default_user_agent(), "elasticsearch-py/%s (Python %s)" % (__versionstr__, python_version())) + self.assertEquals( + con._get_default_user_agent(), + "elasticsearch-py/%s (Python %s)" % (__versionstr__, python_version()), + ) def test_timeout_set(self): con = Urllib3HttpConnection(timeout=42) @@ -254,13 +265,13 @@ class TestUrllib3Connection(TestCase): @patch("elasticsearch.connection.base.logger") def test_uncompressed_body_logged(self, logger): con = self._get_mock_connection(connection_params={"http_compress": True}) - con.perform_request("GET", "/", body=b"{\"example\": \"body\"}") + con.perform_request("GET", "/", body=b'{"example": "body"}') self.assertEquals(2, logger.debug.call_count) req, resp = logger.debug.call_args_list print(req, resp) self.assertEquals('> {"example": "body"}', req[0][0] % req[0][1:]) - self.assertEquals('< {}', resp[0][0] % resp[0][1:]) + self.assertEquals("< {}", resp[0][0] % resp[0][1:]) class TestRequestsConnection(TestCase): @@ -315,7 +326,9 @@ class TestRequestsConnection(TestCase): con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" ) self.assertEquals(con.port, 9243) - self.assertEquals(con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com") + self.assertEquals( + con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com" + ) self.assertTrue(con.http_compress) def test_api_key_auth(self): @@ -324,16 +337,24 @@ class TestRequestsConnection(TestCase): cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", api_key=("elastic", "changeme1"), ) - self.assertEquals(con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=") - self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243") + self.assertEquals( + con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=" + ) + self.assertEquals( + con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + ) # test with base64 encoded string con = RequestsHttpConnection( cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=", ) - self.assertEquals(con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=") - self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243") + self.assertEquals( + con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=" + ) + self.assertEquals( + con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + ) def test_no_http_compression(self): con = self._get_mock_connection() @@ -348,9 +369,7 @@ class TestRequestsConnection(TestCase): self.assertNotIn("accept-encoding", req.headers) def test_http_compression(self): - con = self._get_mock_connection( - {"http_compress": True}, - ) + con = self._get_mock_connection({"http_compress": True},) self.assertTrue(con.http_compress) @@ -380,13 +399,13 @@ class TestRequestsConnection(TestCase): con = RequestsHttpConnection( cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", - http_compress=False + http_compress=False, ) self.assertEquals(con.http_compress, False) con = RequestsHttpConnection( cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", - http_compress=True + http_compress=True, ) self.assertEquals(con.http_compress, True) @@ -442,10 +461,15 @@ class TestRequestsConnection(TestCase): def test_custom_headers(self): con = self._get_mock_connection() - req = self._get_request(con, "GET", "/", headers={ - "content-type": "application/x-ndjson", - "user-agent": "custom-agent/1.2.3", - }) + req = self._get_request( + con, + "GET", + "/", + headers={ + "content-type": "application/x-ndjson", + "user-agent": "custom-agent/1.2.3", + }, + ) self.assertEquals(req.headers["content-type"], "application/x-ndjson") self.assertEquals(req.headers["user-agent"], "custom-agent/1.2.3") @@ -506,7 +530,7 @@ class TestRequestsConnection(TestCase): self.assertEquals(1, logger.warning.call_count) self.assertTrue( re.match( - "^GET http://localhost:9200/\?param=42 \[status:500 request:0.[0-9]{3}s\]", + r"^GET http://localhost:9200/\?param=42 \[status:500 request:0.[0-9]{3}s\]", logger.warning.call_args[0][0] % logger.warning.call_args[0][1:], ) ) @@ -532,7 +556,7 @@ class TestRequestsConnection(TestCase): self.assertEquals(1, tracer.debug.call_count) self.assertTrue( re.match( - '#\[200\] \(0.[0-9]{3}s\)\n#\{\n# "answer": "that\\\\u0027s it!"\n#\}', + r'#\[200\] \(0.[0-9]{3}s\)\n#{\n# "answer": "that\\u0027s it!"\n#}', tracer.debug.call_args[0][0] % tracer.debug.call_args[0][1:], ) ) @@ -541,7 +565,7 @@ class TestRequestsConnection(TestCase): self.assertEquals(1, logger.info.call_count) self.assertTrue( re.match( - "GET http://localhost:9200/\?param=42 \[status:200 request:0.[0-9]{3}s\]", + r"GET http://localhost:9200/\?param=42 \[status:200 request:0.[0-9]{3}s\]", logger.info.call_args[0][0] % logger.info.call_args[0][1:], ) ) @@ -554,12 +578,12 @@ class TestRequestsConnection(TestCase): @patch("elasticsearch.connection.base.logger") def test_uncompressed_body_logged(self, logger): con = self._get_mock_connection(connection_params={"http_compress": True}) - con.perform_request("GET", "/", body=b"{\"example\": \"body\"}") + con.perform_request("GET", "/", body=b'{"example": "body"}') self.assertEquals(2, logger.debug.call_count) req, resp = logger.debug.call_args_list self.assertEquals('> {"example": "body"}', req[0][0] % req[0][1:]) - self.assertEquals('< {}', resp[0][0] % resp[0][1:]) + self.assertEquals("< {}", resp[0][0] % resp[0][1:]) def test_defaults(self): con = self._get_mock_connection() diff --git a/test_elasticsearch/test_serializer.py b/test_elasticsearch/test_serializer.py index 0309c5f1..22e8027d 100644 --- a/test_elasticsearch/test_serializer.py +++ b/test_elasticsearch/test_serializer.py @@ -77,6 +77,6 @@ class TestDeserializer(TestCase): self.assertRaises(SerializationError, self.de.loads, "{}", "text/html") def test_raises_improperly_configured_when_default_mimetype_cannot_be_deserialized( - self + self, ): self.assertRaises(ImproperlyConfigured, Deserializer, {}) diff --git a/test_elasticsearch/test_server/test_common.py b/test_elasticsearch/test_server/test_common.py index fdf7b448..5b83ab40 100644 --- a/test_elasticsearch/test_server/test_common.py +++ b/test_elasticsearch/test_server/test_common.py @@ -24,7 +24,13 @@ PARAMS_RENAMES = {"type": "doc_type", "from": "from_"} CATCH_CODES = {"missing": 404, "conflict": 409, "unauthorized": 401} # test features we have implemented -IMPLEMENTED_FEATURES = {"gtelte", "stash_in_path", "headers", "catch_unauthorized", "default_shards"} +IMPLEMENTED_FEATURES = { + "gtelte", + "stash_in_path", + "headers", + "catch_unauthorized", + "default_shards", +} # broken YAML tests on some releases SKIP_TESTS = { @@ -35,7 +41,7 @@ SKIP_TESTS = { "TestScripts20GetScriptContext", "TestScripts25GetScriptLanguages", # Disallowing expensive queries is 7.7+ - "TestSearch320DisallowQueries" + "TestSearch320DisallowQueries", } } diff --git a/test_elasticsearch/test_transport.py b/test_elasticsearch/test_transport.py index a1fbba9d..a0b87298 100644 --- a/test_elasticsearch/test_transport.py +++ b/test_elasticsearch/test_transport.py @@ -5,7 +5,7 @@ import time from elasticsearch.transport import Transport, get_host_info from elasticsearch.connection import Connection from elasticsearch.connection_pool import DummyConnectionPool -from elasticsearch.exceptions import ConnectionError, ImproperlyConfigured +from elasticsearch.exceptions import ConnectionError from .test_cases import TestCase @@ -107,11 +107,7 @@ class TestTransport(TestCase): self.assertEquals(1, len(t.get_connection().calls)) self.assertEquals(("GET", "/", {}, None), t.get_connection().calls[0][0]) self.assertEquals( - { - "timeout": 42, - "ignore": (), - "headers": None, - }, + {"timeout": 42, "ignore": (), "headers": None}, t.get_connection().calls[0][1], ) @@ -121,7 +117,10 @@ class TestTransport(TestCase): t.perform_request("GET", "/", headers={"user-agent": "my-custom-value/1.2.3"}) self.assertEquals(1, len(t.get_connection().calls)) self.assertEquals( - {"timeout": None, "ignore": (), "headers": {"user-agent": "my-custom-value/1.2.3"} + { + "timeout": None, + "ignore": (), + "headers": {"user-agent": "my-custom-value/1.2.3"}, }, t.get_connection().calls[0][1], ) @@ -323,5 +322,7 @@ class TestTransport(TestCase): ) t.sniff_hosts() # Ensure we parsed out the fqdn and port from the fqdn/ip:port string. - self.assertEqual(t.connection_pool.connection_opts[0][1], - {'host': 'somehost.tld', 'port': 123}) + self.assertEqual( + t.connection_pool.connection_opts[0][1], + {"host": "somehost.tld", "port": 123}, + )