actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[7.x] Surface deprecation warnings from Elasticsearch

Browse Source
This commit is contained in:
Seth Michael Larson
2020-03-31 14:44:20 -05:00
committed by GitHub
parent 606287f041
commit 7f07f1f728
9 changed files with 194 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
elasticsearch/__init__.py
+45 -12
View File
@@ -6,17 +6,7 @@ __version__ = VERSION
__versionstr__ = ".".join(map(str, VERSION))
import logging
try: # Python 2.7+
from logging import NullHandler
except ImportError:
class NullHandler(logging.Handler):
def emit(self, record):
pass
import sys
import warnings
logger = logging.getLogger("elasticsearch")
logger.addHandler(logging.NullHandler())
@@ -26,4 +16,47 @@ from .transport import Transport
from .connection_pool import ConnectionPool, ConnectionSelector, RoundRobinSelector
from .serializer import JSONSerializer
from .connection import Connection, RequestsHttpConnection, Urllib3HttpConnection
from .exceptions import *
from .exceptions import (
ImproperlyConfigured,
ElasticsearchException,
SerializationError,
TransportError,
NotFoundError,
ConflictError,
RequestError,
ConnectionError,
SSLError,
ConnectionTimeout,
AuthenticationException,
AuthorizationException,
ElasticsearchDeprecationWarning,
)
# Only raise one warning per deprecation message so as not
# to spam up the user if the same action is done multiple times.
warnings.simplefilter("default", category=ElasticsearchDeprecationWarning, append=True)
__all__ = [
"Elasticsearch",
"Transport",
"ConnectionPool",
"ConnectionSelector",
"RoundRobinSelector",
"JSONSerializer",
"Connection",
"RequestsHttpConnection",
"Urllib3HttpConnection",
"ImproperlyConfigured",
"ElasticsearchException",
"SerializationError",
"TransportError",
"NotFoundError",
"ConflictError",
"RequestError",
"ConnectionError",
"SSLError",
"ConnectionTimeout",
"AuthenticationException",
"AuthorizationException",
"ElasticsearchDeprecationWarning",
]
elasticsearch/connection/base.py
+39 -1
View File
@@ -2,14 +2,21 @@ import logging
import binascii
import gzip
import io
import re
from platform import python_version
import warnings
try:
import simplejson as json
except ImportError:
import json
from ..exceptions import TransportError, ImproperlyConfigured, HTTP_EXCEPTIONS
from ..exceptions import (
TransportError,
ImproperlyConfigured,
ElasticsearchDeprecationWarning,
HTTP_EXCEPTIONS,
)
from .. import __versionstr__
logger = logging.getLogger("elasticsearch")
@@ -21,6 +28,8 @@ tracer = logging.getLogger("elasticsearch.trace")
if not _tracer_already_configured:
tracer.propagate = False
_WARNING_RE = re.compile(r"\"([^\"]*)\"")
class Connection(object):
"""
@@ -132,6 +141,35 @@ class Connection(object):
f.write(body)
return buf.getvalue()
def _raise_warnings(self, warning_headers):
"""If 'headers' contains a 'Warning' header raise
the warnings to be seen by the user. Takes an iterable
of string values from any number of 'Warning' headers.
"""
if not warning_headers:
return
# Grab only the message from each header, the rest is discarded.
# Format is: '(number) Elasticsearch-(version)-(instance) "(message)"'
warning_messages = []
for header in warning_headers:
# Because 'Requests' does it's own folding of multiple HTTP headers
# into one header delimited by commas (totally standard compliant, just
# annoying for cases like this) we need to expect there may be
# more than one message per 'Warning' header.
matches = _WARNING_RE.findall(header)
if matches:
warning_messages.extend(matches)
else:
# Don't want to throw away any warnings, even if they
# don't follow the format we have now. Use the whole header.
warning_messages.append(header)
for message in warning_messages:
warnings.warn(
message, category=ElasticsearchDeprecationWarning, stacklevel=6
)
def _pretty_json(self, data):
# pretty JSON in tracer curl logs
try:
elasticsearch/connection/http_requests.py
+6
View File
@@ -156,6 +156,12 @@ class RequestsHttpConnection(Connection):
raise ConnectionTimeout("TIMEOUT", str(e), e)
raise ConnectionError("N/A", str(e), e)
# raise warnings if any from the 'Warnings' header.
warnings_headers = (
(response.headers["warning"],) if "warning" in response.headers else ()
)
self._raise_warnings(warnings_headers)
# raise errors based on http status codes, let the client handle those if needed
if (
not (200 <= response.status_code < 300)
elasticsearch/connection/http_urllib3.py
+4
View File
@@ -240,6 +240,10 @@ class Urllib3HttpConnection(Connection):
raise ConnectionTimeout("TIMEOUT", str(e), e)
raise ConnectionError("N/A", str(e), e)
# raise warnings if any from the 'Warnings' header.
warning_headers = response.headers.get_all("warning", ())
self._raise_warnings(warning_headers)
# raise errors based on http status codes, let the client handle those if needed
if not (200 <= response.status < 300) and response.status not in ignore:
self.log_request_fail(
elasticsearch/exceptions.py
+6
View File
@@ -136,6 +136,12 @@ class AuthorizationException(TransportError):
""" Exception representing a 403 status code. """
class ElasticsearchDeprecationWarning(Warning):
""" Warning that is raised when a deprecated option
is flagged via the 'Warning' HTTP header.
"""
# more generic mappings from status_code to python exceptions
HTTP_EXCEPTIONS = {
400: RequestError,
elasticsearch/helpers/test.py
+8 -2
View File
@@ -51,9 +51,15 @@ class ElasticsearchTestCase(TestCase):
def tearDown(self):
super(ElasticsearchTestCase, self).tearDown()
self.client.indices.delete(index="*", ignore=404)
# Hidden indices expanded in wildcards in ES 7.7
expand_wildcards = ["open", "closed"]
if self.es_version >= (7, 7):
expand_wildcards.append("hidden")
self.client.indices.delete(
index="*", ignore=404, expand_wildcards=expand_wildcards
)
self.client.indices.delete_template(name="*", ignore=404)
self.client.indices.delete_alias(index="_all", name="_all", ignore=404)
@property
def es_version(self):
test_elasticsearch/test_connection.py
+44 -1
View File
@@ -4,6 +4,7 @@ import gzip
import io
from mock import Mock, patch
import urllib3
from urllib3._collections import HTTPHeaderDict
import warnings
from requests.auth import AuthBase
from platform import python_version
@@ -87,6 +88,48 @@ class TestBaseConnection(TestCase):
"8af7ee35420f458e903026b4064081f2.westeurope.azure.elastic-cloud.com",
)
def test_empty_warnings(self):
con = Connection()
with warnings.catch_warnings(record=True) as w:
con._raise_warnings(())
con._raise_warnings([])
self.assertEquals(w, [])
def test_raises_warnings(self):
con = Connection()
with warnings.catch_warnings(record=True) as warn:
con._raise_warnings(['299 Elasticsearch-7.6.1-aa751 "this is deprecated"'])
self.assertEquals([str(w.message) for w in warn], ["this is deprecated"])
with warnings.catch_warnings(record=True) as warn:
con._raise_warnings(
[
'299 Elasticsearch-7.6.1-aa751 "this is also deprecated"',
'299 Elasticsearch-7.6.1-aa751 "this is also deprecated"',
'299 Elasticsearch-7.6.1-aa751 "guess what? deprecated"',
]
)
self.assertEquals(
[str(w.message) for w in warn],
["this is also deprecated", "guess what? deprecated"],
)
def test_raises_warnings_when_folded(self):
con = Connection()
with warnings.catch_warnings(record=True) as warn:
con._raise_warnings(
[
'299 Elasticsearch-7.6.1-aa751 "warning",'
'299 Elasticsearch-7.6.1-aa751 "folded"',
]
)
self.assertEquals([str(w.message) for w in warn], ["warning", "folded"])
class TestUrllib3Connection(TestCase):
def _get_mock_connection(self, connection_params={}, response_body=b"{}"):
@@ -94,7 +137,7 @@ class TestUrllib3Connection(TestCase):
def _dummy_urlopen(*args, **kwargs):
dummy_response = Mock()
dummy_response.headers = {}
dummy_response.headers = HTTPHeaderDict({})
dummy_response.status = 200
dummy_response.data = response_body
_dummy_urlopen.call_args = (args, kwargs)
test_elasticsearch/test_server/test_common.py
+35 -12
View File
@@ -8,8 +8,9 @@ from os import walk, environ
from os.path import exists, join, dirname, pardir
import yaml
from shutil import rmtree
import warnings
from elasticsearch import TransportError, RequestError
from elasticsearch import TransportError, RequestError, ElasticsearchDeprecationWarning
from elasticsearch.compat import string_types
from elasticsearch.helpers.test import _get_version
@@ -30,6 +31,7 @@ IMPLEMENTED_FEATURES = {
"headers",
"catch_unauthorized",
"default_shards",
"warnings",
}
# broken YAML tests on some releases
@@ -40,6 +42,8 @@ SKIP_TESTS = {
# Disallowing expensive queries is 7.7+
"TestSearch320DisallowQueries",
"TestIndicesPutIndexTemplate10Basic",
"TestIndicesGetIndexTemplate10Basic",
"TestIndicesGetIndexTemplate20GetMissing",
}
}
@@ -150,6 +154,7 @@ class YamlTestCase(ElasticsearchTestCase):
def run_code(self, test):
""" Execute an instruction based on it's type. """
print(test)
for action in test:
self.assertEquals(1, len(action))
action_type, action = list(action.items())[0]
@@ -164,6 +169,7 @@ class YamlTestCase(ElasticsearchTestCase):
api = self.client
headers = action.pop("headers", None)
catch = action.pop("catch", None)
warn = action.pop("warnings", None)
self.assertEquals(1, len(action))
method, args = list(action.items())[0]
@@ -184,17 +190,34 @@ class YamlTestCase(ElasticsearchTestCase):
for k in args:
args[k] = self._resolve(args[k])
try:
self.last_response = api(**args)
except Exception as e:
if not catch:
raise
self.run_catch(catch, e)
else:
if catch:
raise AssertionError(
"Failed to catch %r in %r." % (catch, self.last_response)
)
warnings.simplefilter("always", category=ElasticsearchDeprecationWarning)
with warnings.catch_warnings(record=True) as caught_warnings:
try:
self.last_response = api(**args)
except Exception as e:
if not catch:
raise
self.run_catch(catch, e)
else:
if catch:
raise AssertionError(
"Failed to catch %r in %r." % (catch, self.last_response)
)
# Filter out warnings raised by other components.
caught_warnings = [
str(w.message)
for w in caught_warnings
if w.category == ElasticsearchDeprecationWarning
]
# Sorting removes the issue with order raised. We only care about
# if all warnings are raised in the single API call.
if warn is not None and sorted(warn) != sorted(caught_warnings):
raise AssertionError(
"Expected warnings not equal to actual warnings: expected=%r actual=%r"
% (warn, caught_warnings)
)
def _get_nodes(self):
if not hasattr(self, "_node_info"):
tox.ini
+7 -2
View File
@@ -7,15 +7,20 @@ setenv =
commands =
python setup.py test
[testenv:lint]
[testenv:blacken]
deps =
flake8
black
commands =
black --target-version=py27 \
elasticsearch/ \
test_elasticsearch/ \
setup.py
[testenv:lint]
deps =
flake8
black
commands =
black --target-version=py27 --check \
elasticsearch/ \
test_elasticsearch/ \