actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Run Black+Flake8 on project

Browse Source
This commit is contained in:
Seth Michael Larson
2020-03-09 11:51:35 -05:00
committed by Seth Michael Larson
parent 3a89b7bd01
commit 210fae23d0
15 changed files with 139 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
elasticsearch/client/__init__.py
+1 -1
View File
@@ -264,7 +264,7 @@ class Elasticsearch(object):
if len(cons) > 5:
cons = cons[:5] + ["..."]
return "<{cls}({cons})>".format(cls=self.__class__.__name__, cons=cons)
except:
except Exception:
# probably operating on custom transport and connection_pool, ignore
return super(Elasticsearch, self).__repr__()
elasticsearch/client/utils.py
+1 -1
View File
@@ -33,7 +33,7 @@ def _escape(value):
# encode strings to utf-8
if isinstance(value, string_types):
if PY2 and isinstance(value, unicode):
if PY2 and isinstance(value, unicode): # noqa: F821
return value.encode("utf-8")
if not PY2 and isinstance(value, str):
return value.encode("utf-8")
elasticsearch/compat.py
+12 -1
View File
@@ -3,7 +3,7 @@ import sys
PY2 = sys.version_info[0] == 2
if PY2:
string_types = (basestring,)
string_types = (basestring,) # noqa: F821
from urllib import quote_plus, quote, urlencode, unquote
from urlparse import urlparse
from itertools import imap as map
@@ -14,3 +14,14 @@ else:
map = map
from queue import Queue
__all__ = [
"string_types",
"quote_plus",
"quote",
"urlencode",
"unquote",
"urlparse",
"map",
"Queue",
]
elasticsearch/connection/base.py
+6 -6
View File
@@ -57,7 +57,9 @@ class Connection(object):
try:
_, cloud_id = cloud_id.split(":")
parent_dn, es_uuid, _ = (
binascii.a2b_base64(cloud_id.encode("utf-8")).decode("utf-8").split("$")
binascii.a2b_base64(cloud_id.encode("utf-8"))
.decode("utf-8")
.split("$")
)
except ValueError:
raise ImproperlyConfigured("'cloud_id' is not properly formatted")
@@ -106,9 +108,7 @@ class Connection(object):
def __eq__(self, other):
if not isinstance(other, Connection):
raise TypeError(
"Unsupported equality check for %s and %s" % (self, other)
)
raise TypeError("Unsupported equality check for %s and %s" % (self, other))
return self.__hash__() == other.__hash__()
def __hash__(self):
@@ -242,6 +242,6 @@ class Connection(object):
:arg api_key, either a tuple or a base64 encoded string
"""
if isinstance(api_key, (tuple, list)):
s = "{0}:{1}".format(api_key[0], api_key[1]).encode('utf-8')
return "ApiKey " + binascii.b2a_base64(s).rstrip(b"\r\n").decode('utf-8')
s = "{0}:{1}".format(api_key[0], api_key[1]).encode("utf-8")
return "ApiKey " + binascii.b2a_base64(s).rstrip(b"\r\n").decode("utf-8")
return "ApiKey " + api_key
elasticsearch/connection/http_urllib3.py
+10 -14
View File
@@ -5,6 +5,15 @@ from urllib3.exceptions import ReadTimeoutError, SSLError as UrllibSSLError
from urllib3.util.retry import Retry
import warnings
from .base import Connection
from ..exceptions import (
ConnectionError,
ImproperlyConfigured,
ConnectionTimeout,
SSLError,
)
from ..compat import urlencode
# sentinel value for `verify_certs` and `ssl_show_warn`.
# This is used to detect if a user is passing in a value
# for SSL kwargs if also using an SSLContext.
@@ -20,15 +29,6 @@ try:
except ImportError:
pass
from .base import Connection
from ..exceptions import (
ConnectionError,
ImproperlyConfigured,
ConnectionTimeout,
SSLError,
)
from ..compat import urlencode
def create_ssl_context(**kwargs):
"""
@@ -188,11 +188,7 @@ class Urllib3HttpConnection(Connection):
urllib3.disable_warnings()
self.pool = pool_class(
self.hostname,
port=self.port,
timeout=self.timeout,
maxsize=maxsize,
**kw
self.hostname, port=self.port, timeout=self.timeout, maxsize=maxsize, **kw
)
def perform_request(
elasticsearch/connection_pool.py
+4 -1
View File
@@ -150,7 +150,10 @@ class ConnectionPool(object):
try:
self.connections.remove(connection)
except ValueError:
logger.info("Attempted to remove %r, but it does not exist in the connection pool.", connection)
logger.info(
"Attempted to remove %r, but it does not exist in the connection pool.",
connection,
)
# connection not alive or another thread marked it already, ignore
return
else:
elasticsearch/helpers/__init__.py
+13
View File
@@ -2,3 +2,16 @@ from .errors import BulkIndexError, ScanError
from .actions import expand_action, streaming_bulk, bulk, parallel_bulk
from .actions import scan, reindex
from .actions import _chunk_actions, _process_bulk_chunk
__all__ = [
"BulkIndexError",
"ScanError",
"expand_action",
"streaming_bulk",
"bulk",
"parallel_bulk",
"scan",
"reindex",
"_chunk_actions",
"_process_bulk_chunk",
]
elasticsearch/helpers/actions.py
+7 -1
View File
@@ -45,7 +45,13 @@ def expand_action(data):
"version_type",
):
if key in data:
if key in ["_parent", "_retry_on_conflict", "_routing", "_version", "_version_type"]:
if key in [
"_parent",
"_retry_on_conflict",
"_routing",
"_version",
"_version_type",
]:
action[op_type][key[1:]] = data.pop(key)
else:
action[op_type][key] = data.pop(key)
elasticsearch/helpers/test.py
+4 -1
View File
@@ -12,7 +12,10 @@ def get_test_client(nowait=False, **kwargs):
if "PYTHON_CONNECTION_CLASS" in os.environ:
from elasticsearch import connection
kw["connection_class"] = getattr(connection, os.environ["PYTHON_CONNECTION_CLASS"])
kw["connection_class"] = getattr(
connection, os.environ["PYTHON_CONNECTION_CLASS"]
)
kw.update(kwargs)
client = Elasticsearch([os.environ.get("ELASTICSEARCH_HOST", {})], **kw)
elasticsearch/transport.py
+4 -6
View File
@@ -1,11 +1,9 @@
import time
from itertools import chain
from platform import python_version
from .connection import Urllib3HttpConnection
from .connection_pool import ConnectionPool, DummyConnectionPool
from .serializer import JSONSerializer, Deserializer, DEFAULT_SERIALIZERS
from . import __versionstr__
from .exceptions import (
ConnectionError,
TransportError,
@@ -230,7 +228,7 @@ class Transport(object):
pass
else:
raise TransportError("N/A", "Unable to sniff hosts.")
except:
except Exception:
# keep the previous value on error
self.last_sniff = previous_sniff
raise
@@ -245,11 +243,11 @@ class Transport(object):
if not address or ":" not in address:
return None
if '/' in address:
if "/" in address:
# Support 7.x host/ip:port behavior where http.publish_host has been set.
fqdn, ipaddress = address.split('/', 1)
fqdn, ipaddress = address.split("/", 1)
host["host"] = fqdn
_, host["port"] = ipaddress.rsplit(':', 1)
_, host["port"] = ipaddress.rsplit(":", 1)
host["port"] = int(host["port"])
else:
test_elasticsearch/test_cases.py
+2 -7
View File
@@ -1,11 +1,6 @@
from collections import defaultdict
try:
# python 2.6
from unittest2 import TestCase, SkipTest
except ImportError:
from unittest import TestCase, SkipTest
from unittest import TestCase
from unittest import SkipTest # noqa: F401
from elasticsearch import Elasticsearch
test_elasticsearch/test_connection.py
+56 -32
View File
@@ -25,9 +25,7 @@ def gzip_decompress(data):
class TestUrllib3Connection(TestCase):
def _get_mock_connection(
self, connection_params={}, response_body=b"{}"
):
def _get_mock_connection(self, connection_params={}, response_body=b"{}"):
con = Urllib3HttpConnection(**connection_params)
def _dummy_urlopen(*args, **kwargs):
@@ -66,7 +64,9 @@ class TestUrllib3Connection(TestCase):
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
)
self.assertEquals(con.port, 9243)
self.assertEquals(con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com")
self.assertEquals(
con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com"
)
self.assertTrue(con.http_compress)
def test_api_key_auth(self):
@@ -75,16 +75,24 @@ class TestUrllib3Connection(TestCase):
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
api_key=("elastic", "changeme1"),
)
self.assertEquals(con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=")
self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243")
self.assertEquals(
con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
)
# test with base64 encoded string
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=",
)
self.assertEquals(con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=")
self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243")
self.assertEquals(
con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
)
def test_no_http_compression(self):
con = self._get_mock_connection()
@@ -134,19 +142,22 @@ class TestUrllib3Connection(TestCase):
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
http_compress=False
http_compress=False,
)
self.assertEquals(con.http_compress, False)
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
http_compress=True
http_compress=True,
)
self.assertEquals(con.http_compress, True)
def test_default_user_agent(self):
con = Urllib3HttpConnection()
self.assertEquals(con._get_default_user_agent(), "elasticsearch-py/%s (Python %s)" % (__versionstr__, python_version()))
self.assertEquals(
con._get_default_user_agent(),
"elasticsearch-py/%s (Python %s)" % (__versionstr__, python_version()),
)
def test_timeout_set(self):
con = Urllib3HttpConnection(timeout=42)
@@ -254,13 +265,13 @@ class TestUrllib3Connection(TestCase):
@patch("elasticsearch.connection.base.logger")
def test_uncompressed_body_logged(self, logger):
con = self._get_mock_connection(connection_params={"http_compress": True})
con.perform_request("GET", "/", body=b"{\"example\": \"body\"}")
con.perform_request("GET", "/", body=b'{"example": "body"}')
self.assertEquals(2, logger.debug.call_count)
req, resp = logger.debug.call_args_list
print(req, resp)
self.assertEquals('> {"example": "body"}', req[0][0] % req[0][1:])
self.assertEquals('< {}', resp[0][0] % resp[0][1:])
self.assertEquals("< {}", resp[0][0] % resp[0][1:])
class TestRequestsConnection(TestCase):
@@ -315,7 +326,9 @@ class TestRequestsConnection(TestCase):
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
)
self.assertEquals(con.port, 9243)
self.assertEquals(con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com")
self.assertEquals(
con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com"
)
self.assertTrue(con.http_compress)
def test_api_key_auth(self):
@@ -324,16 +337,24 @@ class TestRequestsConnection(TestCase):
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
api_key=("elastic", "changeme1"),
)
self.assertEquals(con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=")
self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243")
self.assertEquals(
con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
)
# test with base64 encoded string
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=",
)
self.assertEquals(con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=")
self.assertEquals(con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243")
self.assertEquals(
con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
)
def test_no_http_compression(self):
con = self._get_mock_connection()
@@ -348,9 +369,7 @@ class TestRequestsConnection(TestCase):
self.assertNotIn("accept-encoding", req.headers)
def test_http_compression(self):
con = self._get_mock_connection(
{"http_compress": True},
)
con = self._get_mock_connection({"http_compress": True},)
self.assertTrue(con.http_compress)
@@ -380,13 +399,13 @@ class TestRequestsConnection(TestCase):
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
http_compress=False
http_compress=False,
)
self.assertEquals(con.http_compress, False)
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
http_compress=True
http_compress=True,
)
self.assertEquals(con.http_compress, True)
@@ -442,10 +461,15 @@ class TestRequestsConnection(TestCase):
def test_custom_headers(self):
con = self._get_mock_connection()
req = self._get_request(con, "GET", "/", headers={
"content-type": "application/x-ndjson",
"user-agent": "custom-agent/1.2.3",
})
req = self._get_request(
con,
"GET",
"/",
headers={
"content-type": "application/x-ndjson",
"user-agent": "custom-agent/1.2.3",
},
)
self.assertEquals(req.headers["content-type"], "application/x-ndjson")
self.assertEquals(req.headers["user-agent"], "custom-agent/1.2.3")
@@ -506,7 +530,7 @@ class TestRequestsConnection(TestCase):
self.assertEquals(1, logger.warning.call_count)
self.assertTrue(
re.match(
"^GET http://localhost:9200/\?param=42 \[status:500 request:0.[0-9]{3}s\]",
r"^GET http://localhost:9200/\?param=42 \[status:500 request:0.[0-9]{3}s\]",
logger.warning.call_args[0][0] % logger.warning.call_args[0][1:],
)
)
@@ -532,7 +556,7 @@ class TestRequestsConnection(TestCase):
self.assertEquals(1, tracer.debug.call_count)
self.assertTrue(
re.match(
'#\[200\] \(0.[0-9]{3}s\)\n#\{\n# "answer": "that\\\\u0027s it!"\n#\}',
r'#\[200\] \(0.[0-9]{3}s\)\n#{\n# "answer": "that\\u0027s it!"\n#}',
tracer.debug.call_args[0][0] % tracer.debug.call_args[0][1:],
)
)
@@ -541,7 +565,7 @@ class TestRequestsConnection(TestCase):
self.assertEquals(1, logger.info.call_count)
self.assertTrue(
re.match(
"GET http://localhost:9200/\?param=42 \[status:200 request:0.[0-9]{3}s\]",
r"GET http://localhost:9200/\?param=42 \[status:200 request:0.[0-9]{3}s\]",
logger.info.call_args[0][0] % logger.info.call_args[0][1:],
)
)
@@ -554,12 +578,12 @@ class TestRequestsConnection(TestCase):
@patch("elasticsearch.connection.base.logger")
def test_uncompressed_body_logged(self, logger):
con = self._get_mock_connection(connection_params={"http_compress": True})
con.perform_request("GET", "/", body=b"{\"example\": \"body\"}")
con.perform_request("GET", "/", body=b'{"example": "body"}')
self.assertEquals(2, logger.debug.call_count)
req, resp = logger.debug.call_args_list
self.assertEquals('> {"example": "body"}', req[0][0] % req[0][1:])
self.assertEquals('< {}', resp[0][0] % resp[0][1:])
self.assertEquals("< {}", resp[0][0] % resp[0][1:])
def test_defaults(self):
con = self._get_mock_connection()
test_elasticsearch/test_serializer.py
+1 -1
View File
@@ -77,6 +77,6 @@ class TestDeserializer(TestCase):
self.assertRaises(SerializationError, self.de.loads, "{}", "text/html")
def test_raises_improperly_configured_when_default_mimetype_cannot_be_deserialized(
self
self,
):
self.assertRaises(ImproperlyConfigured, Deserializer, {})
test_elasticsearch/test_server/test_common.py
+8 -2
View File
@@ -24,7 +24,13 @@ PARAMS_RENAMES = {"type": "doc_type", "from": "from_"}
CATCH_CODES = {"missing": 404, "conflict": 409, "unauthorized": 401}
# test features we have implemented
IMPLEMENTED_FEATURES = {"gtelte", "stash_in_path", "headers", "catch_unauthorized", "default_shards"}
IMPLEMENTED_FEATURES = {
"gtelte",
"stash_in_path",
"headers",
"catch_unauthorized",
"default_shards",
}
# broken YAML tests on some releases
SKIP_TESTS = {
@@ -35,7 +41,7 @@ SKIP_TESTS = {
"TestScripts20GetScriptContext",
"TestScripts25GetScriptLanguages",
# Disallowing expensive queries is 7.7+
"TestSearch320DisallowQueries"
"TestSearch320DisallowQueries",
}
}
test_elasticsearch/test_transport.py
+10 -9
View File
@@ -5,7 +5,7 @@ import time
from elasticsearch.transport import Transport, get_host_info
from elasticsearch.connection import Connection
from elasticsearch.connection_pool import DummyConnectionPool
from elasticsearch.exceptions import ConnectionError, ImproperlyConfigured
from elasticsearch.exceptions import ConnectionError
from .test_cases import TestCase
@@ -107,11 +107,7 @@ class TestTransport(TestCase):
self.assertEquals(1, len(t.get_connection().calls))
self.assertEquals(("GET", "/", {}, None), t.get_connection().calls[0][0])
self.assertEquals(
{
"timeout": 42,
"ignore": (),
"headers": None,
},
{"timeout": 42, "ignore": (), "headers": None},
t.get_connection().calls[0][1],
)
@@ -121,7 +117,10 @@ class TestTransport(TestCase):
t.perform_request("GET", "/", headers={"user-agent": "my-custom-value/1.2.3"})
self.assertEquals(1, len(t.get_connection().calls))
self.assertEquals(
{"timeout": None, "ignore": (), "headers": {"user-agent": "my-custom-value/1.2.3"}
{
"timeout": None,
"ignore": (),
"headers": {"user-agent": "my-custom-value/1.2.3"},
},
t.get_connection().calls[0][1],
)
@@ -323,5 +322,7 @@ class TestTransport(TestCase):
)
t.sniff_hosts()
# Ensure we parsed out the fqdn and port from the fqdn/ip:port string.
self.assertEqual(t.connection_pool.connection_opts[0][1],
{'host': 'somehost.tld', 'port': 123})
self.assertEqual(
t.connection_pool.connection_opts[0][1],
{"host": "somehost.tld", "port": 123},
)