actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Remove metadata header check.

Browse Source 
This commit is essentially a revert of following three commits:

fe5367382b
58b376159c
b894e359df

Signed-off-by: Rushi Agrawal <[email protected]>
This commit is contained in:
Rushi Agrawal
2021-08-19 08:42:20 +05:30
parent 004e346bb0
commit ed73459f05
20 changed files with 27 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
opensearch/_async/helpers.py
-1
View File
@@ -407,7 +407,6 @@ async def async_scan(
body={"scroll_id": [scroll_id]},
**transport_kwargs,
ignore=(404,),
params={"__elastic_client_meta": (("h", "s"),)},
)
opensearch/_async/http_aiohttp.py
-4
View File
@@ -39,7 +39,6 @@ from ..exceptions import (
ImproperlyConfigured,
SSLError,
)
from ..utils import _client_meta_version
from ._extra_imports import aiohttp, aiohttp_exceptions, yarl
from .compat import get_running_loop
@@ -79,9 +78,6 @@ class AsyncConnection(Connection):
class AIOHttpConnection(AsyncConnection):
HTTP_CLIENT_META = ("ai", _client_meta_version(aiohttp.__version__))
def __init__(
self,
host="localhost",
opensearch/_async/http_aiohttp.pyi
+3 -4
View File
@@ -24,7 +24,7 @@
# specific language governing permissions and limitations
# under the License.
from typing import Any, Collection, Mapping, MutableMapping, Optional, Tuple, Union
from typing import Any, Collection, Mapping, Optional, Tuple, Union
from ..connection import Connection
from ._extra_imports import aiohttp # type: ignore
@@ -34,11 +34,11 @@ class AsyncConnection(Connection):
self,
method: str,
url: str,
params: Optional[MutableMapping[str, Any]] = ...,
params: Optional[Mapping[str, Any]] = ...,
body: Optional[bytes] = ...,
timeout: Optional[Union[int, float]] = ...,
ignore: Collection[int] = ...,
headers: Optional[MutableMapping[str, str]] = ...,
headers: Optional[Mapping[str, str]] = ...,
) -> Tuple[int, Mapping[str, str], str]: ...
async def close(self) -> None: ...
@@ -65,7 +65,6 @@ class AIOHttpConnection(AsyncConnection):
cloud_id: Optional[str] = ...,
api_key: Optional[Any] = ...,
opaque_id: Optional[str] = ...,
meta_header: bool = ...,
loop: Any = ...,
**kwargs: Any,
) -> None: ...
opensearch/_async/transport.py
+2 -2
View File
@@ -332,8 +332,8 @@ class AsyncTransport(Transport):
"""
await self._async_call()
method, headers, params, body, ignore, timeout = self._resolve_request_args(
method, headers, params, body
method, params, body, ignore, timeout = self._resolve_request_args(
method, params, body
)
for attempt in range(self.max_retries + 1):
opensearch/connection/base.py
-7
View File
@@ -77,8 +77,6 @@ class Connection(object):
For tracing all requests made by this transport.
"""
HTTP_CLIENT_META = None
def __init__(
self,
host="localhost",
@@ -91,7 +89,6 @@ class Connection(object):
cloud_id=None,
api_key=None,
opaque_id=None,
meta_header=True,
**kwargs
):
@@ -168,10 +165,6 @@ class Connection(object):
self.url_prefix = url_prefix
self.timeout = timeout
if not isinstance(meta_header, bool):
raise TypeError("meta_header must be of type bool")
self.meta_header = meta_header
def __repr__(self):
return "<%s: %s>" % (self.__class__.__name__, self.host)
opensearch/connection/base.pyi
+3 -5
View File
@@ -27,11 +27,11 @@
import logging
from typing import (
Any,
AnyStr,
Collection,
Dict,
List,
Mapping,
MutableMapping,
NoReturn,
Optional,
Sequence,
@@ -52,7 +52,6 @@ class Connection(object):
host: str
url_prefix: str
timeout: Optional[Union[float, int]]
meta_header: bool
def __init__(
self,
host: str = ...,
@@ -65,7 +64,6 @@ class Connection(object):
cloud_id: Optional[str] = ...,
api_key: Optional[Union[Tuple[str, str], List[str], str]] = ...,
opaque_id: Optional[str] = ...,
meta_header: bool = ...,
**kwargs: Any
) -> None: ...
def __repr__(self) -> str: ...
@@ -87,11 +85,11 @@ class Connection(object):
self,
method: str,
url: str,
params: Optional[MutableMapping[str, Any]] = ...,
params: Optional[Mapping[str, Any]] = ...,
body: Optional[bytes] = ...,
timeout: Optional[Union[int, float]] = ...,
ignore: Collection[int] = ...,
headers: Optional[MutableMapping[str, str]] = ...,
headers: Optional[Mapping[str, str]] = ...,
) -> Tuple[int, Mapping[str, str], str]: ...
def log_request_success(
self,
opensearch/connection/http_requests.py
+8 -13
View File
@@ -27,6 +27,13 @@
import time
import warnings
try:
import requests
REQUESTS_AVAILABLE = True
except ImportError:
REQUESTS_AVAILABLE = False
from ..compat import reraise_exceptions, string_types, urlencode
from ..exceptions import (
ConnectionError,
@@ -34,18 +41,8 @@ from ..exceptions import (
ImproperlyConfigured,
SSLError,
)
from ..utils import _client_meta_version
from .base import Connection
try:
import requests
REQUESTS_AVAILABLE = True
_REQUESTS_META_VERSION = _client_meta_version(requests.__version__)
except ImportError:
REQUESTS_AVAILABLE = False
_REQUESTS_META_VERSION = ""
class RequestsHttpConnection(Connection):
"""
@@ -71,8 +68,6 @@ class RequestsHttpConnection(Connection):
For tracing all requests made by this transport.
"""
HTTP_CLIENT_META = ("rq", _REQUESTS_META_VERSION)
def __init__(
self,
host="localhost",
@@ -156,7 +151,7 @@ class RequestsHttpConnection(Connection):
url = self.base_url + url
headers = headers or {}
if params:
url = "%s?%s" % (url, urlencode(params))
url = "%s?%s" % (url, urlencode(params or {}))
orig_body = body
if self.http_compress and body:
opensearch/connection/http_requests.pyi
-1
View File
@@ -48,6 +48,5 @@ class RequestsHttpConnection(Connection):
cloud_id: Optional[str] = ...,
api_key: Optional[Any] = ...,
opaque_id: Optional[str] = ...,
meta_header: bool = ...,
**kwargs: Any
) -> None: ...
opensearch/connection/http_urllib3.py
-4
View File
@@ -40,7 +40,6 @@ from ..exceptions import (
ImproperlyConfigured,
SSLError,
)
from ..utils import _client_meta_version
from .base import Connection
# sentinel value for `verify_certs` and `ssl_show_warn`.
@@ -108,8 +107,6 @@ class Urllib3HttpConnection(Connection):
For tracing all requests made by this transport.
"""
HTTP_CLIENT_META = ("ur", _client_meta_version(urllib3.__version__))
def __init__(
self,
host="localhost",
@@ -234,7 +231,6 @@ class Urllib3HttpConnection(Connection):
url = "%s?%s" % (url, urlencode(params))
full_url = self.host + url
start = time.time()
orig_body = body
try:
opensearch/connection/http_urllib3.pyi
-1
View File
@@ -62,6 +62,5 @@ class Urllib3HttpConnection(Connection):
cloud_id: Optional[str] = ...,
api_key: Optional[Any] = ...,
opaque_id: Optional[str] = ...,
meta_header: bool = ...,
**kwargs: Any
) -> None: ...
opensearch/helpers/actions.py
+1 -14
View File
@@ -239,8 +239,6 @@ def _process_bulk_chunk(
"""
Send a bulk request to opensearch and process the output.
"""
kwargs = _add_helper_meta_to_kwargs(kwargs, "bp")
if not isinstance(ignore_status, (list, tuple)):
ignore_status = (ignore_status,)
@@ -266,13 +264,6 @@ def _process_bulk_chunk(
yield item
def _add_helper_meta_to_kwargs(kwargs, helper_meta):
params = (kwargs or {}).pop("params", {})
params["__elastic_client_meta"] = (("h", helper_meta),)
kwargs["params"] = params
return kwargs
def streaming_bulk(
client,
actions,
@@ -553,7 +544,6 @@ def scan(
"""
scroll_kwargs = scroll_kwargs or {}
_add_helper_meta_to_kwargs(scroll_kwargs, "s")
if not preserve_order:
query = query.copy() if query else {}
@@ -616,10 +606,7 @@ def scan(
finally:
if scroll_id and clear_scroll:
client.clear_scroll(
body={"scroll_id": [scroll_id]},
ignore=(404,),
params={"__elastic_client_meta": (("h", "s"),)},
**transport_kwargs
body={"scroll_id": [scroll_id]}, ignore=(404,), **transport_kwargs
)
opensearch/transport.py
+4 -37
View File
@@ -26,9 +26,7 @@
import time
from itertools import chain
from platform import python_version
from ._version import __versionstr__
from .connection import Urllib3HttpConnection
from .connection_pool import ConnectionPool, DummyConnectionPool, EmptyConnectionPool
from .exceptions import (
@@ -38,7 +36,6 @@ from .exceptions import (
TransportError,
)
from .serializer import DEFAULT_SERIALIZERS, Deserializer, JSONSerializer
from .utils import _client_meta_version
def get_host_info(node_info, host):
@@ -88,7 +85,6 @@ class Transport(object):
retry_on_status=(502, 503, 504),
retry_on_timeout=False,
send_get_body_as="GET",
meta_header=True,
**kwargs
):
"""
@@ -123,8 +119,6 @@ class Transport(object):
don't support passing bodies with GET requests. If you set this to
'POST' a POST method will be used instead, if to 'source' then the body
will be serialized and passed as a query parameter `source`.
:arg meta_header: If True will send the 'X-Elastic-Client-Meta' HTTP header containing
simple client metadata. Setting to False will disable the header. Defaults to True.
Any extra keyword arguments will be passed to the `connection_class`
when creating and instance unless overridden by that connection's
@@ -132,8 +126,6 @@ class Transport(object):
"""
if connection_class is None:
connection_class = self.DEFAULT_CONNECTION_CLASS
if not isinstance(meta_header, bool):
raise TypeError("meta_header must be of type bool")
# serialization config
_serializers = DEFAULT_SERIALIZERS.copy()
@@ -149,7 +141,6 @@ class Transport(object):
self.retry_on_timeout = retry_on_timeout
self.retry_on_status = retry_on_status
self.send_get_body_as = send_get_body_as
self.meta_header = meta_header
# data serializer
self.serializer = serializer
@@ -193,20 +184,6 @@ class Transport(object):
if sniff_on_start:
self.sniff_hosts(True)
# Create the default metadata for the x-elastic-client-meta
# HTTP header. Only requires adding the (service, service_version)
# tuple to the beginning of the client_meta
self._client_meta = (
("es", _client_meta_version(__versionstr__)),
("py", _client_meta_version(python_version())),
("t", _client_meta_version(__versionstr__)),
)
# Grab the 'HTTP_CLIENT_META' property from the connection class
http_client_meta = getattr(connection_class, "HTTP_CLIENT_META", None)
if http_client_meta:
self._client_meta += (http_client_meta,)
def add_connection(self, host):
"""
Create a new :class:`~opensearch.Connection` instance and add it to the pool.
@@ -385,8 +362,8 @@ class Transport(object):
:arg body: body of the request, will be serialized using serializer and
passed to the connection
"""
method, headers, params, body, ignore, timeout = self._resolve_request_args(
method, headers, params, body
method, params, body, ignore, timeout = self._resolve_request_args(
method, params, body
)
for attempt in range(self.max_retries + 1):
@@ -453,7 +430,7 @@ class Transport(object):
"""
self.connection_pool.close()
def _resolve_request_args(self, method, headers, params, body):
def _resolve_request_args(self, method, params, body):
"""Resolves parameters for .perform_request()"""
if body is not None:
body = self.serializer.dumps(body)
@@ -485,15 +462,5 @@ class Transport(object):
ignore = params.pop("ignore", ())
if isinstance(ignore, int):
ignore = (ignore,)
client_meta = params.pop("__elastic_client_meta", ())
else:
client_meta = ()
if self.meta_header:
headers = headers or {}
client_meta = self._client_meta + client_meta
headers["x-elastic-client-meta"] = ",".join(
"%s=%s" % (k, v) for k, v in client_meta
)
return method, headers, params, body, ignore, timeout
return method, params, body, ignore, timeout
opensearch/transport.pyi
-1
View File
@@ -76,7 +76,6 @@ class Transport(object):
retry_on_status: Collection[int] = ...,
retry_on_timeout: bool = ...,
send_get_body_as: str = ...,
meta_header: bool = ...,
**kwargs: Any
) -> None: ...
def add_connection(self, host: Any) -> None: ...
opensearch/utils.py
-40
View File
@@ -1,40 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
#
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import re
def _client_meta_version(version):
"""Transforms a Python package version to one
compatible with 'X-Elastic-Client-Meta'. Essentially
replaces any pre-release information with a 'p' suffix.
"""
version, version_pre = re.match(
r"^([0-9][0-9.]*[0-9]|[0-9])(.*)$", version
).groups()
if version_pre:
version += "p"
return version
opensearch/utils.pyi
-27
View File
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
#
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
def _client_meta_version(version: str) -> str: ...
test_opensearch/test_async/test_transport.py
+3 -39
View File
@@ -29,7 +29,6 @@ from __future__ import unicode_literals
import asyncio
import json
import re
import pytest
from mock import patch
@@ -129,7 +128,7 @@ class TestTransport:
assert isinstance(t.connection_pool, DummyConnectionPool)
async def test_request_timeout_extracted_from_params_and_passed(self):
t = AsyncTransport([{}], connection_class=DummyConnection, meta_header=False)
t = AsyncTransport([{}], connection_class=DummyConnection)
await t.perform_request("GET", "/", params={"request_timeout": 42})
assert 1 == len(t.get_connection().calls)
@@ -141,9 +140,7 @@ class TestTransport:
} == t.get_connection().calls[0][1]
async def test_opaque_id(self):
t = AsyncTransport(
[{}], opaque_id="app-1", connection_class=DummyConnection, meta_header=False
)
t = AsyncTransport([{}], opaque_id="app-1", connection_class=DummyConnection)
await t.perform_request("GET", "/")
assert 1 == len(t.get_connection().calls)
@@ -165,7 +162,7 @@ class TestTransport:
} == t.get_connection().calls[1][1]
async def test_request_with_custom_user_agent_header(self):
t = AsyncTransport([{}], connection_class=DummyConnection, meta_header=False)
t = AsyncTransport([{}], connection_class=DummyConnection)
await t.perform_request(
"GET", "/", headers={"user-agent": "my-custom-value/1.2.3"}
@@ -195,39 +192,6 @@ class TestTransport:
assert 1 == len(t.get_connection().calls)
assert ("POST", "/", None, b"{}") == t.get_connection().calls[0][0]
async def test_client_meta_header(self):
t = AsyncTransport([{}], connection_class=DummyConnection)
await t.perform_request("GET", "/", body={})
assert len(t.get_connection().calls) == 1
headers = t.get_connection().calls[0][1]["headers"]
assert re.match(
r"^es=[0-9.]+p?,py=[0-9.]+p?,t=[0-9.]+p?$",
headers["x-elastic-client-meta"],
)
class DummyConnectionWithMeta(DummyConnection):
HTTP_CLIENT_META = ("dm", "1.2.3")
t = AsyncTransport([{}], connection_class=DummyConnectionWithMeta)
await t.perform_request("GET", "/", body={}, headers={"Custom": "header"})
assert len(t.get_connection().calls) == 1
headers = t.get_connection().calls[0][1]["headers"]
assert re.match(
r"^es=[0-9.]+p?,py=[0-9.]+p?,t=[0-9.]+p?,dm=1.2.3$",
headers["x-elastic-client-meta"],
)
assert headers["Custom"] == "header"
async def test_client_meta_header_not_sent(self):
t = AsyncTransport([{}], meta_header=False, connection_class=DummyConnection)
await t.perform_request("GET", "/", body={})
assert len(t.get_connection().calls) == 1
headers = t.get_connection().calls[0][1]["headers"]
assert headers is None
async def test_body_gets_encoded_into_bytes(self):
t = AsyncTransport([{}], connection_class=DummyConnection)
test_opensearch/test_connection.py
-10
View File
@@ -174,16 +174,6 @@ class TestBaseConnection(TestCase):
conn = Connection(**kwargs)
assert conn.host == expected_host
def test_meta_header(self):
conn = Connection(meta_header=True)
assert conn.meta_header is True
conn = Connection(meta_header=False)
assert conn.meta_header is False
with pytest.raises(TypeError) as e:
Connection(meta_header=1)
assert str(e.value) == "meta_header must be of type bool"
def test_compatibility_accept_header(self):
try:
conn = Connection()
test_opensearch/test_helpers.py
-15
View File
@@ -32,7 +32,6 @@ import mock
import pytest
from opensearch import OpenSearch, helpers
from opensearch.helpers import actions
from opensearch.serializer import JSONSerializer
from .test_cases import TestCase
@@ -214,20 +213,6 @@ class TestChunkActions(TestCase):
chunk = chunk if isinstance(chunk, str) else chunk.encode("utf-8")
self.assertLessEqual(len(chunk), max_byte_size)
def test_add_helper_meta_to_kwargs(self):
self.assertEqual(
actions._add_helper_meta_to_kwargs({}, "b"),
{"params": {"__elastic_client_meta": (("h", "b"),)}},
)
self.assertEqual(
actions._add_helper_meta_to_kwargs({"params": {}}, "b"),
{"params": {"__elastic_client_meta": (("h", "b"),)}},
)
self.assertEqual(
actions._add_helper_meta_to_kwargs({"params": {"key": "value"}}, "b"),
{"params": {"__elastic_client_meta": (("h", "b"),), "key": "value"}},
)
class TestExpandActions(TestCase):
def test_string_actions_are_marked_as_simple_inserts(self):
test_opensearch/test_transport.py
+3 -43
View File
@@ -30,7 +30,6 @@ from __future__ import unicode_literals
import json
import time
import pytest
from mock import patch
from opensearch.connection import Connection
@@ -132,7 +131,7 @@ class TestTransport(TestCase):
self.assertIsInstance(t.connection_pool, DummyConnectionPool)
def test_request_timeout_extracted_from_params_and_passed(self):
t = Transport([{}], meta_header=False, connection_class=DummyConnection)
t = Transport([{}], connection_class=DummyConnection)
t.perform_request("GET", "/", params={"request_timeout": 42})
self.assertEqual(1, len(t.get_connection().calls))
@@ -143,9 +142,7 @@ class TestTransport(TestCase):
)
def test_opaque_id(self):
t = Transport(
[{}], opaque_id="app-1", meta_header=False, connection_class=DummyConnection
)
t = Transport([{}], opaque_id="app-1", connection_class=DummyConnection)
t.perform_request("GET", "/")
self.assertEqual(1, len(t.get_connection().calls))
@@ -165,7 +162,7 @@ class TestTransport(TestCase):
)
def test_request_with_custom_user_agent_header(self):
t = Transport([{}], meta_header=False, connection_class=DummyConnection)
t = Transport([{}], connection_class=DummyConnection)
t.perform_request("GET", "/", headers={"user-agent": "my-custom-value/1.2.3"})
self.assertEqual(1, len(t.get_connection().calls))
@@ -194,43 +191,6 @@ class TestTransport(TestCase):
self.assertEqual(1, len(t.get_connection().calls))
self.assertEqual(("POST", "/", None, b"{}"), t.get_connection().calls[0][0])
def test_client_meta_header(self):
t = Transport([{}], connection_class=DummyConnection)
t.perform_request("GET", "/", body={})
self.assertEqual(1, len(t.get_connection().calls))
headers = t.get_connection().calls[0][1]["headers"]
self.assertRegexpMatches(
headers["x-elastic-client-meta"], r"^es=[0-9.]+p?,py=[0-9.]+p?,t=[0-9.]+p?$"
)
class DummyConnectionWithMeta(DummyConnection):
HTTP_CLIENT_META = ("dm", "1.2.3")
t = Transport([{}], connection_class=DummyConnectionWithMeta)
t.perform_request("GET", "/", body={}, headers={"Custom": "header"})
self.assertEqual(1, len(t.get_connection().calls))
headers = t.get_connection().calls[0][1]["headers"]
self.assertRegexpMatches(
headers["x-elastic-client-meta"],
r"^es=[0-9.]+p?,py=[0-9.]+p?,t=[0-9.]+p?,dm=1.2.3$",
)
self.assertEqual(headers["Custom"], "header")
def test_client_meta_header_not_sent(self):
t = Transport([{}], meta_header=False, connection_class=DummyConnection)
t.perform_request("GET", "/", body={})
self.assertEqual(1, len(t.get_connection().calls))
headers = t.get_connection().calls[0][1]["headers"]
self.assertIs(headers, None)
def test_meta_header_type_error(self):
with pytest.raises(TypeError) as e:
Transport([{}], meta_header=1)
assert str(e.value) == "meta_header must be of type bool"
def test_body_gets_encoded_into_bytes(self):
t = Transport([{}], connection_class=DummyConnection)
test_opensearch/test_utils.py
-37
View File
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
#
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import pytest
from opensearch.utils import _client_meta_version
@pytest.mark.parametrize(
["version", "meta_version"],
[("1.26.3", "1.26.3"), ("7.10.1a1", "7.10.1p"), ("7.10.pre", "7.10p")],
)
def test_client_meta_version(version, meta_version):
assert _client_meta_version(version) == meta_version