Rename module to opensearchpy

To avoid conflict with an existing package by name 'opensearch' being
present

Signed-off-by: Rushi Agrawal <[email protected]>

test_opensearchpy/__init__.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.

test_opensearchpy/run_tests.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from __future__ import print_function
+
+import subprocess
+import sys
+from os import environ
+from os.path import abspath, dirname, exists, join, pardir
+
+
+def fetch_opensearch_repo():
+    # user is manually setting YAML dir, don't tamper with it
+    if "TEST_OPENSEARCH_YAML_DIR" in environ:
+        return
+
+    repo_path = environ.get(
+        "TEST_OPENSEARCH_REPO",
+        abspath(join(dirname(__file__), pardir, pardir, "opensearch")),
+    )
+
+    # no repo
+    if not exists(repo_path) or not exists(join(repo_path, ".git")):
+        subprocess.check_call(
+            "git clone https://github.com/opensearch-project/opensearch %s" % repo_path,
+            shell=True,
+        )
+
+    # set YAML test dir
+    environ["TEST_OPENSEARCH_YAML_DIR"] = join(
+        repo_path, "rest-api-spec", "src", "main", "resources", "rest-api-spec", "test"
+    )
+
+    # fetching of yaml tests disabled, we'll run with what's there
+    if environ.get("TEST_OPENSEARCH_NOFETCH", False):
+        return
+
+    from test_opensearchpy.test_cases import SkipTest
+    from test_opensearchpy.test_server import get_client
+
+    # find out the sha of the running client
+    try:
+        client = get_client()
+        sha = client.info()["version"]["build_hash"]
+    except (SkipTest, KeyError):
+        print("No running opensearch >1.X server...")
+        return
+
+    # fetch new commits to be sure...
+    print("Fetching opensearch repo...")
+    subprocess.check_call(
+        "cd %s && git fetch https://github.com/opensearch-project/opensearch.git"
+        % repo_path,
+        shell=True,
+    )
+    # reset to the version from info()
+    subprocess.check_call("cd %s && git fetch" % repo_path, shell=True)
+    subprocess.check_call("cd %s && git reset --hard %s" % (repo_path, sha), shell=True)
+
+
+def run_all(argv=None):
+    sys.exitfunc = lambda: sys.stderr.write("Shutting down....\n")
+
+    # fetch yaml tests anywhere that's not GitHub Actions
+    if "GITHUB_ACTION" not in environ:
+        fetch_opensearch_repo()
+
+    # always insert coverage when running tests
+    if argv is None:
+        junit_xml = join(
+            abspath(dirname(dirname(__file__))), "junit", "opensearch-py-junit.xml"
+        )
+        argv = [
+            "pytest",
+            "--cov=opensearch",
+            "--junitxml=%s" % junit_xml,
+            "--log-level=DEBUG",
+            "--cache-clear",
+            "-vv",
+        ]
+
+        secured = False
+        if environ.get("OPENSEARCH_URL", "").startswith("https://"):
+            secured = True
+
+        ignores = []
+        # Python 3.6+ is required for async
+        if sys.version_info < (3, 6):
+            ignores.append("test_opensearchpy/test_async/")
+
+        # GitHub Actions, run non-server tests
+        if "GITHUB_ACTION" in environ:
+            ignores.extend(
+                [
+                    "test_opensearchpy/test_server/",
+                    "test_opensearchpy/test_server_secured/",
+                    "test_opensearchpy/test_async/test_server/",
+                ]
+            )
+
+        # Jenkins/Github actions, only run server tests
+        if environ.get("TEST_TYPE") == "server":
+            test_dir = abspath(dirname(__file__))
+            if secured:
+                argv.append(join(test_dir, "test_server_secured"))
+                ignores.extend(
+                    [
+                        "test_opensearchpy/test_server/",
+                        "test_opensearchpy/test_async/test_server/",
+                    ]
+                )
+            else:
+                argv.append(join(test_dir, "test_server"))
+                if sys.version_info >= (3, 6):
+                    argv.append(join(test_dir, "test_async/test_server"))
+                ignores.extend(
+                    [
+                        "test_opensearchpy/test_server_secured/",
+                    ]
+                )
+
+        if ignores:
+            argv.extend(["--ignore=%s" % ignore for ignore in ignores])
+
+        # Not in CI, run all tests specified.
+        else:
+            argv.append(abspath(dirname(__file__)))
+
+    exit_code = 0
+    try:
+        subprocess.check_call(argv, stdout=sys.stdout, stderr=sys.stderr)
+    except subprocess.CalledProcessError as e:
+        exit_code = e.returncode
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    run_all(sys.argv)

test_opensearchpy/test_async/__init__.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.

test_opensearchpy/test_async/test_connection.py

@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import gzip
+import io
+import json
+import ssl
+import warnings
+from platform import python_version
+
+import aiohttp
+import pytest
+from mock import patch
+from multidict import CIMultiDict
+
+from opensearchpy import AIOHttpConnection, __versionstr__
+from opensearchpy.compat import reraise_exceptions
+from opensearchpy.exceptions import ConnectionError
+
+pytestmark = pytest.mark.asyncio
+
+
+def gzip_decompress(data):
+    buf = gzip.GzipFile(fileobj=io.BytesIO(data), mode="rb")
+    return buf.read()
+
+
+class TestAIOHttpConnection:
+    async def _get_mock_connection(self, connection_params={}, response_body=b"{}"):
+        con = AIOHttpConnection(**connection_params)
+        await con._create_aiohttp_session()
+
+        def _dummy_request(*args, **kwargs):
+            class DummyResponse:
+                async def __aenter__(self, *_, **__):
+                    return self
+
+                async def __aexit__(self, *_, **__):
+                    pass
+
+                async def text(self):
+                    return response_body.decode("utf-8", "surrogatepass")
+
+            dummy_response = DummyResponse()
+            dummy_response.headers = CIMultiDict()
+            dummy_response.status = 200
+            _dummy_request.call_args = (args, kwargs)
+            return dummy_response
+
+        con.session.request = _dummy_request
+        return con
+
+    async def test_ssl_context(self):
+        try:
+            context = ssl.create_default_context()
+        except AttributeError:
+            # if create_default_context raises an AttributeError Exception
+            # it means SSLContext is not available for that version of python
+            # and we should skip this test.
+            pytest.skip(
+                "Test test_ssl_context is skipped cause SSLContext is not available for this version of Python"
+            )
+
+        con = AIOHttpConnection(use_ssl=True, ssl_context=context)
+        await con._create_aiohttp_session()
+        assert con.use_ssl
+        assert con.session.connector._ssl == context
+
+    def test_opaque_id(self):
+        con = AIOHttpConnection(opaque_id="app-1")
+        assert con.headers["x-opaque-id"] == "app-1"
+
+    def test_http_cloud_id(self):
+        con = AIOHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng=="
+        )
+        assert con.use_ssl
+        assert (
+            con.host
+            == "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        )
+        assert con.port is None
+        assert con.hostname == "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        assert con.http_compress
+
+        con = AIOHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+            port=9243,
+        )
+        assert (
+            con.host
+            == "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io:9243"
+        )
+        assert con.port == 9243
+        assert con.hostname == "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+
+    def test_api_key_auth(self):
+        # test with tuple
+        con = AIOHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+            api_key=("elastic", "changeme1"),
+        )
+        assert con.headers["authorization"] == "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE="
+        assert (
+            con.host
+            == "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        )
+
+        # test with base64 encoded string
+        con = AIOHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+            api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=",
+        )
+        assert con.headers["authorization"] == "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI="
+        assert (
+            con.host
+            == "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        )
+
+    async def test_no_http_compression(self):
+        con = await self._get_mock_connection()
+        assert not con.http_compress
+        assert "accept-encoding" not in con.headers
+
+        await con.perform_request("GET", "/")
+
+        _, kwargs = con.session.request.call_args
+
+        assert not kwargs["data"]
+        assert "accept-encoding" not in kwargs["headers"]
+        assert "content-encoding" not in kwargs["headers"]
+
+    async def test_http_compression(self):
+        con = await self._get_mock_connection({"http_compress": True})
+        assert con.http_compress
+        assert con.headers["accept-encoding"] == "gzip,deflate"
+
+        # 'content-encoding' shouldn't be set at a connection level.
+        # Should be applied only if the request is sent with a body.
+        assert "content-encoding" not in con.headers
+
+        await con.perform_request("GET", "/", body=b"{}")
+
+        _, kwargs = con.session.request.call_args
+
+        assert gzip_decompress(kwargs["data"]) == b"{}"
+        assert kwargs["headers"]["accept-encoding"] == "gzip,deflate"
+        assert kwargs["headers"]["content-encoding"] == "gzip"
+
+        await con.perform_request("GET", "/")
+
+        _, kwargs = con.session.request.call_args
+
+        assert not kwargs["data"]
+        assert kwargs["headers"]["accept-encoding"] == "gzip,deflate"
+        assert "content-encoding" not in kwargs["headers"]
+
+    def test_cloud_id_http_compress_override(self):
+        # 'http_compress' will be 'True' by default for connections with
+        # 'cloud_id' set but should prioritize user-defined values.
+        con = AIOHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+        )
+        assert con.http_compress is True
+
+        con = AIOHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+            http_compress=False,
+        )
+        assert con.http_compress is False
+
+        con = AIOHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+            http_compress=True,
+        )
+        assert con.http_compress is True
+
+    async def test_url_prefix(self):
+        con = await self._get_mock_connection(
+            connection_params={"url_prefix": "/_search/"}
+        )
+        assert con.url_prefix == "/_search"
+
+        await con.perform_request("GET", "/")
+
+        # Need to convert the yarl URL to a string to compare.
+        method, yarl_url = con.session.request.call_args[0]
+        assert method == "GET" and str(yarl_url) == "http://localhost:9200/_search/"
+
+    def test_default_user_agent(self):
+        con = AIOHttpConnection()
+        assert con._get_default_user_agent() == "opensearch-py/%s (Python %s)" % (
+            __versionstr__,
+            python_version(),
+        )
+
+    def test_timeout_set(self):
+        con = AIOHttpConnection(timeout=42)
+        assert 42 == con.timeout
+
+    def test_keep_alive_is_on_by_default(self):
+        con = AIOHttpConnection()
+        assert {
+            "connection": "keep-alive",
+            "content-type": "application/json",
+            "user-agent": con._get_default_user_agent(),
+        } == con.headers
+
+    def test_http_auth(self):
+        con = AIOHttpConnection(http_auth="username:secret")
+        assert {
+            "authorization": "Basic dXNlcm5hbWU6c2VjcmV0",
+            "connection": "keep-alive",
+            "content-type": "application/json",
+            "user-agent": con._get_default_user_agent(),
+        } == con.headers
+
+    def test_http_auth_tuple(self):
+        con = AIOHttpConnection(http_auth=("username", "secret"))
+        assert {
+            "authorization": "Basic dXNlcm5hbWU6c2VjcmV0",
+            "content-type": "application/json",
+            "connection": "keep-alive",
+            "user-agent": con._get_default_user_agent(),
+        } == con.headers
+
+    def test_http_auth_list(self):
+        con = AIOHttpConnection(http_auth=["username", "secret"])
+        assert {
+            "authorization": "Basic dXNlcm5hbWU6c2VjcmV0",
+            "content-type": "application/json",
+            "connection": "keep-alive",
+            "user-agent": con._get_default_user_agent(),
+        } == con.headers
+
+    def test_uses_https_if_verify_certs_is_off(self):
+        with warnings.catch_warnings(record=True) as w:
+            con = AIOHttpConnection(use_ssl=True, verify_certs=False)
+            assert 1 == len(w)
+            assert (
+                "Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure."
+                == str(w[0].message)
+            )
+
+        assert con.use_ssl
+        assert con.scheme == "https"
+        assert con.host == "https://localhost:9200"
+
+    async def test_nowarn_when_test_uses_https_if_verify_certs_is_off(self):
+        with warnings.catch_warnings(record=True) as w:
+            con = AIOHttpConnection(
+                use_ssl=True, verify_certs=False, ssl_show_warn=False
+            )
+            await con._create_aiohttp_session()
+            assert w == []
+
+        assert isinstance(con.session, aiohttp.ClientSession)
+
+    def test_doesnt_use_https_if_not_specified(self):
+        con = AIOHttpConnection()
+        assert not con.use_ssl
+
+    def test_no_warning_when_using_ssl_context(self):
+        ctx = ssl.create_default_context()
+        with warnings.catch_warnings(record=True) as w:
+            AIOHttpConnection(ssl_context=ctx)
+            assert w == [], str([x.message for x in w])
+
+    def test_warns_if_using_non_default_ssl_kwargs_with_ssl_context(self):
+        for kwargs in (
+            {"ssl_show_warn": False},
+            {"ssl_show_warn": True},
+            {"verify_certs": True},
+            {"verify_certs": False},
+            {"ca_certs": "/path/to/certs"},
+            {"ssl_show_warn": True, "ca_certs": "/path/to/certs"},
+        ):
+            kwargs["ssl_context"] = ssl.create_default_context()
+
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter("always")
+
+                AIOHttpConnection(**kwargs)
+
+                assert 1 == len(w)
+                assert (
+                    "When using `ssl_context`, all other SSL related kwargs are ignored"
+                    == str(w[0].message)
+                )
+
+    @patch("opensearchpy.connection.base.logger")
+    async def test_uncompressed_body_logged(self, logger):
+        con = await self._get_mock_connection(connection_params={"http_compress": True})
+        await con.perform_request("GET", "/", body=b'{"example": "body"}')
+
+        assert 2 == logger.debug.call_count
+        req, resp = logger.debug.call_args_list
+
+        assert '> {"example": "body"}' == req[0][0] % req[0][1:]
+        assert "< {}" == resp[0][0] % resp[0][1:]
+
+    async def test_surrogatepass_into_bytes(self):
+        buf = b"\xe4\xbd\xa0\xe5\xa5\xbd\xed\xa9\xaa"
+        con = await self._get_mock_connection(response_body=buf)
+        status, headers, data = await con.perform_request("GET", "/")
+        assert u"你好\uda6a" == data
+
+    @pytest.mark.parametrize("exception_cls", reraise_exceptions)
+    async def test_recursion_error_reraised(self, exception_cls):
+        conn = AIOHttpConnection()
+
+        def request_raise(*_, **__):
+            raise exception_cls("Wasn't modified!")
+
+        await conn._create_aiohttp_session()
+        conn.session.request = request_raise
+
+        with pytest.raises(exception_cls) as e:
+            await conn.perform_request("GET", "/")
+        assert str(e.value) == "Wasn't modified!"
+
+
+class TestConnectionHttpbin:
+    """Tests the HTTP connection implementations against a live server E2E"""
+
+    async def httpbin_anything(self, conn, **kwargs):
+        status, headers, data = await conn.perform_request("GET", "/anything", **kwargs)
+        data = json.loads(data)
+        data["headers"].pop(
+            "X-Amzn-Trace-Id", None
+        )  # Remove this header as it's put there by AWS.
+        return (status, data)
+
+    async def test_aiohttp_connection(self):
+        # Defaults
+        conn = AIOHttpConnection("httpbin.org", port=443, use_ssl=True)
+        user_agent = conn._get_default_user_agent()
+        status, data = await self.httpbin_anything(conn)
+        assert status == 200
+        assert data["method"] == "GET"
+        assert data["headers"] == {
+            "Content-Type": "application/json",
+            "Host": "httpbin.org",
+            "User-Agent": user_agent,
+        }
+
+        # http_compress=False
+        conn = AIOHttpConnection(
+            "httpbin.org", port=443, use_ssl=True, http_compress=False
+        )
+        status, data = await self.httpbin_anything(conn)
+        assert status == 200
+        assert data["method"] == "GET"
+        assert data["headers"] == {
+            "Content-Type": "application/json",
+            "Host": "httpbin.org",
+            "User-Agent": user_agent,
+        }
+
+        # http_compress=True
+        conn = AIOHttpConnection(
+            "httpbin.org", port=443, use_ssl=True, http_compress=True
+        )
+        status, data = await self.httpbin_anything(conn)
+        assert status == 200
+        assert data["headers"] == {
+            "Accept-Encoding": "gzip,deflate",
+            "Content-Type": "application/json",
+            "Host": "httpbin.org",
+            "User-Agent": user_agent,
+        }
+
+        # Headers
+        conn = AIOHttpConnection(
+            "httpbin.org",
+            port=443,
+            use_ssl=True,
+            http_compress=True,
+            headers={"header1": "value1"},
+        )
+        status, data = await self.httpbin_anything(
+            conn, headers={"header2": "value2", "header1": "override!"}
+        )
+        assert status == 200
+        assert data["headers"] == {
+            "Accept-Encoding": "gzip,deflate",
+            "Content-Type": "application/json",
+            "Host": "httpbin.org",
+            "Header1": "override!",
+            "Header2": "value2",
+            "User-Agent": user_agent,
+        }
+
+    async def test_aiohttp_connection_error(self):
+        conn = AIOHttpConnection("not.a.host.name")
+        with pytest.raises(ConnectionError):
+            await conn.perform_request("GET", "/")

test_opensearchpy/test_async/test_server/__init__.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.

test_opensearchpy/test_async/test_server/conftest.py

@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import asyncio
+
+import pytest
+
+import opensearchpy
+from opensearchpy.helpers.test import CA_CERTS, OPENSEARCH_URL
+
+from ...utils import wipe_cluster
+
+pytestmark = pytest.mark.asyncio
+
+
+@pytest.fixture(scope="function")
+async def async_client():
+    client = None
+    try:
+        if not hasattr(opensearchpy, "AsyncOpenSearch"):
+            pytest.skip("test requires 'AsyncOpenSearch'")
+
+        kw = {"timeout": 3, "ca_certs": CA_CERTS}
+        client = opensearchpy.AsyncOpenSearch(OPENSEARCH_URL, **kw)
+
+        # wait for yellow status
+        for _ in range(100):
+            try:
+                await client.cluster.health(wait_for_status="yellow")
+                break
+            except ConnectionError:
+                await asyncio.sleep(0.1)
+        else:
+            # timeout
+            pytest.skip("OpenSearch failed to start.")
+
+        yield client
+
+    finally:
+        if client:
+            wipe_cluster(client)
+            await client.close()

test_opensearchpy/test_async/test_server/test_clients.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from __future__ import unicode_literals
+
+import pytest
+
+pytestmark = pytest.mark.asyncio
+
+
+class TestUnicode:
+    async def test_indices_analyze(self, async_client):
+        await async_client.indices.analyze(body='{"text": "привет"}')
+
+
+class TestBulk:
+    async def test_bulk_works_with_string_body(self, async_client):
+        docs = '{ "index" : { "_index" : "bulk_test_index", "_id" : "1" } }\n{"answer": 42}'
+        response = await async_client.bulk(body=docs)
+
+        assert response["errors"] is False
+        assert len(response["items"]) == 1
+
+    async def test_bulk_works_with_bytestring_body(self, async_client):
+        docs = b'{ "index" : { "_index" : "bulk_test_index", "_id" : "2" } }\n{"answer": 42}'
+        response = await async_client.bulk(body=docs)
+
+        assert response["errors"] is False
+        assert len(response["items"]) == 1
+
+
+class TestYarlMissing:
+    async def test_aiohttp_connection_works_without_yarl(
+        self, async_client, monkeypatch
+    ):
+        # This is a defensive test case for if aiohttp suddenly stops using yarl.
+        from opensearchpy._async import http_aiohttp
+
+        monkeypatch.setattr(http_aiohttp, "yarl", False)
+
+        resp = await async_client.info(pretty=True)
+        assert isinstance(resp, dict)

test_opensearchpy/test_async/test_server/test_helpers.py

@@ -0,0 +1,901 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+# Licensed to Elasticsearch B.V.under one or more agreements.
+# Elasticsearch B.V.licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information
+
+import asyncio
+
+import pytest
+from mock import MagicMock, patch
+
+from opensearchpy import TransportError, helpers
+from opensearchpy.helpers import ScanError
+
+pytestmark = pytest.mark.asyncio
+
+
+class AsyncMock(MagicMock):
+    async def __call__(self, *args, **kwargs):
+        return super(AsyncMock, self).__call__(*args, **kwargs)
+
+    def __await__(self):
+        return self().__await__()
+
+
+class FailingBulkClient(object):
+    def __init__(
+        self, client, fail_at=(2,), fail_with=TransportError(599, "Error!", {})
+    ):
+        self.client = client
+        self._called = 0
+        self._fail_at = fail_at
+        self.transport = client.transport
+        self._fail_with = fail_with
+
+    async def bulk(self, *args, **kwargs):
+        self._called += 1
+        if self._called in self._fail_at:
+            raise self._fail_with
+        return await self.client.bulk(*args, **kwargs)
+
+
+class TestStreamingBulk(object):
+    async def test_actions_remain_unchanged(self, async_client):
+        actions = [{"_id": 1}, {"_id": 2}]
+        async for ok, item in helpers.async_streaming_bulk(
+            async_client, actions, index="test-index"
+        ):
+            assert ok
+        assert [{"_id": 1}, {"_id": 2}] == actions
+
+    async def test_all_documents_get_inserted(self, async_client):
+        docs = [{"answer": x, "_id": x} for x in range(100)]
+        async for ok, item in helpers.async_streaming_bulk(
+            async_client, docs, index="test-index", refresh=True
+        ):
+            assert ok
+
+        assert 100 == (await async_client.count(index="test-index"))["count"]
+        assert {"answer": 42} == (await async_client.get(index="test-index", id=42))[
+            "_source"
+        ]
+
+    async def test_documents_data_types(self, async_client):
+        async def async_gen():
+            for x in range(100):
+                await asyncio.sleep(0)
+                yield {"answer": x, "_id": x}
+
+        def sync_gen():
+            for x in range(100):
+                yield {"answer": x, "_id": x}
+
+        async for ok, item in helpers.async_streaming_bulk(
+            async_client, async_gen(), index="test-index", refresh=True
+        ):
+            assert ok
+
+        assert 100 == (await async_client.count(index="test-index"))["count"]
+        assert {"answer": 42} == (await async_client.get(index="test-index", id=42))[
+            "_source"
+        ]
+
+        await async_client.delete_by_query(
+            index="test-index", body={"query": {"match_all": {}}}
+        )
+
+        async for ok, item in helpers.async_streaming_bulk(
+            async_client, sync_gen(), index="test-index", refresh=True
+        ):
+            assert ok
+
+        assert 100 == (await async_client.count(index="test-index"))["count"]
+        assert {"answer": 42} == (await async_client.get(index="test-index", id=42))[
+            "_source"
+        ]
+
+    async def test_all_errors_from_chunk_are_raised_on_failure(self, async_client):
+        await async_client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        await async_client.cluster.health(wait_for_status="yellow")
+
+        try:
+            async for ok, item in helpers.async_streaming_bulk(
+                async_client, [{"a": "b"}, {"a": "c"}], index="i", raise_on_error=True
+            ):
+                assert ok
+        except helpers.BulkIndexError as e:
+            assert 2 == len(e.errors)
+        else:
+            assert False, "exception should have been raised"
+
+    async def test_different_op_types(self, async_client):
+        await async_client.index(index="i", id=45, body={})
+        await async_client.index(index="i", id=42, body={})
+        docs = [
+            {"_index": "i", "_id": 47, "f": "v"},
+            {"_op_type": "delete", "_index": "i", "_id": 45},
+            {"_op_type": "update", "_index": "i", "_id": 42, "doc": {"answer": 42}},
+        ]
+        async for ok, item in helpers.async_streaming_bulk(async_client, docs):
+            assert ok
+
+        assert not await async_client.exists(index="i", id=45)
+        assert {"answer": 42} == (await async_client.get(index="i", id=42))["_source"]
+        assert {"f": "v"} == (await async_client.get(index="i", id=47))["_source"]
+
+    async def test_transport_error_can_becaught(self, async_client):
+        failing_client = FailingBulkClient(async_client)
+        docs = [
+            {"_index": "i", "_id": 47, "f": "v"},
+            {"_index": "i", "_id": 45, "f": "v"},
+            {"_index": "i", "_id": 42, "f": "v"},
+        ]
+
+        results = [
+            x
+            async for x in helpers.async_streaming_bulk(
+                failing_client,
+                docs,
+                raise_on_exception=False,
+                raise_on_error=False,
+                chunk_size=1,
+            )
+        ]
+        assert 3 == len(results)
+        assert [True, False, True] == [r[0] for r in results]
+
+        exc = results[1][1]["index"].pop("exception")
+        assert isinstance(exc, TransportError)
+        assert 599 == exc.status_code
+        assert {
+            "index": {
+                "_index": "i",
+                "_id": 45,
+                "data": {"f": "v"},
+                "error": "TransportError(599, 'Error!')",
+                "status": 599,
+            }
+        } == results[1][1]
+
+    async def test_rejected_documents_are_retried(self, async_client):
+        failing_client = FailingBulkClient(
+            async_client, fail_with=TransportError(429, "Rejected!", {})
+        )
+        docs = [
+            {"_index": "i", "_id": 47, "f": "v"},
+            {"_index": "i", "_id": 45, "f": "v"},
+            {"_index": "i", "_id": 42, "f": "v"},
+        ]
+        results = [
+            x
+            async for x in helpers.async_streaming_bulk(
+                failing_client,
+                docs,
+                raise_on_exception=False,
+                raise_on_error=False,
+                chunk_size=1,
+                max_retries=1,
+                initial_backoff=0,
+            )
+        ]
+        assert 3 == len(results)
+        assert [True, True, True] == [r[0] for r in results]
+        await async_client.indices.refresh(index="i")
+        res = await async_client.search(index="i")
+        assert {"value": 3, "relation": "eq"} == res["hits"]["total"]
+        assert 4 == failing_client._called
+
+    async def test_rejected_documents_are_retried_at_most_max_retries_times(
+        self, async_client
+    ):
+        failing_client = FailingBulkClient(
+            async_client, fail_at=(1, 2), fail_with=TransportError(429, "Rejected!", {})
+        )
+
+        docs = [
+            {"_index": "i", "_id": 47, "f": "v"},
+            {"_index": "i", "_id": 45, "f": "v"},
+            {"_index": "i", "_id": 42, "f": "v"},
+        ]
+        results = [
+            x
+            async for x in helpers.async_streaming_bulk(
+                failing_client,
+                docs,
+                raise_on_exception=False,
+                raise_on_error=False,
+                chunk_size=1,
+                max_retries=1,
+                initial_backoff=0,
+            )
+        ]
+        assert 3 == len(results)
+        assert [False, True, True] == [r[0] for r in results]
+        await async_client.indices.refresh(index="i")
+        res = await async_client.search(index="i")
+        assert {"value": 2, "relation": "eq"} == res["hits"]["total"]
+        assert 4 == failing_client._called
+
+    async def test_transport_error_is_raised_with_max_retries(self, async_client):
+        failing_client = FailingBulkClient(
+            async_client,
+            fail_at=(1, 2, 3, 4),
+            fail_with=TransportError(429, "Rejected!", {}),
+        )
+
+        async def streaming_bulk():
+            results = [
+                x
+                async for x in helpers.async_streaming_bulk(
+                    failing_client,
+                    [{"a": 42}, {"a": 39}],
+                    raise_on_exception=True,
+                    max_retries=3,
+                    initial_backoff=0,
+                )
+            ]
+            return results
+
+        with pytest.raises(TransportError):
+            await streaming_bulk()
+        assert 4 == failing_client._called
+
+
+class TestBulk(object):
+    async def test_bulk_works_with_single_item(self, async_client):
+        docs = [{"answer": 42, "_id": 1}]
+        success, failed = await helpers.async_bulk(
+            async_client, docs, index="test-index", refresh=True
+        )
+
+        assert 1 == success
+        assert not failed
+        assert 1 == (await async_client.count(index="test-index"))["count"]
+        assert {"answer": 42} == (await async_client.get(index="test-index", id=1))[
+            "_source"
+        ]
+
+    async def test_all_documents_get_inserted(self, async_client):
+        docs = [{"answer": x, "_id": x} for x in range(100)]
+        success, failed = await helpers.async_bulk(
+            async_client, docs, index="test-index", refresh=True
+        )
+
+        assert 100 == success
+        assert not failed
+        assert 100 == (await async_client.count(index="test-index"))["count"]
+        assert {"answer": 42} == (await async_client.get(index="test-index", id=42))[
+            "_source"
+        ]
+
+    async def test_stats_only_reports_numbers(self, async_client):
+        docs = [{"answer": x} for x in range(100)]
+        success, failed = await helpers.async_bulk(
+            async_client, docs, index="test-index", refresh=True, stats_only=True
+        )
+
+        assert 100 == success
+        assert 0 == failed
+        assert 100 == (await async_client.count(index="test-index"))["count"]
+
+    async def test_errors_are_reported_correctly(self, async_client):
+        await async_client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        await async_client.cluster.health(wait_for_status="yellow")
+
+        success, failed = await helpers.async_bulk(
+            async_client,
+            [{"a": 42}, {"a": "c", "_id": 42}],
+            index="i",
+            raise_on_error=False,
+        )
+        assert 1 == success
+        assert 1 == len(failed)
+        error = failed[0]
+        assert "42" == error["index"]["_id"]
+        assert "i" == error["index"]["_index"]
+        print(error["index"]["error"])
+        assert "MapperParsingException" in repr(
+            error["index"]["error"]
+        ) or "mapper_parsing_exception" in repr(error["index"]["error"])
+
+    async def test_error_is_raised(self, async_client):
+        await async_client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        await async_client.cluster.health(wait_for_status="yellow")
+
+        with pytest.raises(helpers.BulkIndexError):
+            await helpers.async_bulk(async_client, [{"a": 42}, {"a": "c"}], index="i")
+
+    async def test_ignore_error_if_raised(self, async_client):
+        # ignore the status code 400 in tuple
+        await helpers.async_bulk(
+            async_client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=(400,)
+        )
+
+        # ignore the status code 400 in list
+        await helpers.async_bulk(
+            async_client,
+            [{"a": 42}, {"a": "c"}],
+            index="i",
+            ignore_status=[
+                400,
+            ],
+        )
+
+        # ignore the status code 400
+        await helpers.async_bulk(
+            async_client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=400
+        )
+
+        # ignore only the status code in the `ignore_status` argument
+        with pytest.raises(helpers.BulkIndexError):
+            await helpers.async_bulk(
+                async_client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=(444,)
+            )
+
+        # ignore transport error exception
+        failing_client = FailingBulkClient(async_client)
+        await helpers.async_bulk(
+            failing_client, [{"a": 42}], index="i", ignore_status=(599,)
+        )
+
+    async def test_errors_are_collected_properly(self, async_client):
+        await async_client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        await async_client.cluster.health(wait_for_status="yellow")
+
+        success, failed = await helpers.async_bulk(
+            async_client,
+            [{"a": 42}, {"a": "c"}],
+            index="i",
+            stats_only=True,
+            raise_on_error=False,
+        )
+        assert 1 == success
+        assert 1 == failed
+
+
+class MockScroll:
+    def __init__(self):
+        self.calls = []
+
+    async def __call__(self, *args, **kwargs):
+        self.calls.append((args, kwargs))
+        if len(self.calls) == 1:
+            return {
+                "_scroll_id": "dummy_id",
+                "_shards": {"successful": 4, "total": 5, "skipped": 0},
+                "hits": {"hits": [{"scroll_data": 42}]},
+            }
+        elif len(self.calls) == 2:
+            return {
+                "_scroll_id": "dummy_id",
+                "_shards": {"successful": 4, "total": 5, "skipped": 0},
+                "hits": {"hits": []},
+            }
+        else:
+            raise Exception("no more responses")
+
+
+class MockResponse:
+    def __init__(self, resp):
+        self.resp = resp
+
+    async def __call__(self, *args, **kwargs):
+        return self.resp
+
+    def __await__(self):
+        return self().__await__()
+
+
+@pytest.fixture(scope="function")
+async def scan_teardown(async_client):
+    yield
+    await async_client.clear_scroll(scroll_id="_all")
+
+
+class TestScan(object):
+    async def test_order_can_be_preserved(self, async_client, scan_teardown):
+        bulk = []
+        for x in range(100):
+            bulk.append({"index": {"_index": "test_index", "_id": x}})
+            bulk.append({"answer": x, "correct": x == 42})
+        await async_client.bulk(bulk, refresh=True)
+
+        docs = [
+            doc
+            async for doc in helpers.async_scan(
+                async_client,
+                index="test_index",
+                query={"sort": "answer"},
+                preserve_order=True,
+            )
+        ]
+
+        assert 100 == len(docs)
+        assert list(map(str, range(100))) == list(d["_id"] for d in docs)
+        assert list(range(100)) == list(d["_source"]["answer"] for d in docs)
+
+    async def test_all_documents_are_read(self, async_client, scan_teardown):
+        bulk = []
+        for x in range(100):
+            bulk.append({"index": {"_index": "test_index", "_id": x}})
+            bulk.append({"answer": x, "correct": x == 42})
+        await async_client.bulk(bulk, refresh=True)
+
+        docs = [
+            x
+            async for x in helpers.async_scan(async_client, index="test_index", size=2)
+        ]
+
+        assert 100 == len(docs)
+        assert set(map(str, range(100))) == set(d["_id"] for d in docs)
+        assert set(range(100)) == set(d["_source"]["answer"] for d in docs)
+
+    async def test_scroll_error(self, async_client, scan_teardown):
+        bulk = []
+        for x in range(4):
+            bulk.append({"index": {"_index": "test_index"}})
+            bulk.append({"value": x})
+        await async_client.bulk(bulk, refresh=True)
+
+        with patch.object(async_client, "scroll", MockScroll()):
+            data = [
+                x
+                async for x in helpers.async_scan(
+                    async_client,
+                    index="test_index",
+                    size=2,
+                    raise_on_error=False,
+                    clear_scroll=False,
+                )
+            ]
+            assert len(data) == 3
+            assert data[-1] == {"scroll_data": 42}
+
+        with patch.object(async_client, "scroll", MockScroll()):
+            with pytest.raises(ScanError):
+                data = [
+                    x
+                    async for x in helpers.async_scan(
+                        async_client,
+                        index="test_index",
+                        size=2,
+                        raise_on_error=True,
+                        clear_scroll=False,
+                    )
+                ]
+            assert len(data) == 3
+            assert data[-1] == {"scroll_data": 42}
+
+    async def test_initial_search_error(self, async_client, scan_teardown):
+        with patch.object(async_client, "clear_scroll", new_callable=AsyncMock):
+            with patch.object(
+                async_client,
+                "search",
+                MockResponse(
+                    {
+                        "_scroll_id": "dummy_id",
+                        "_shards": {"successful": 4, "total": 5, "skipped": 0},
+                        "hits": {"hits": [{"search_data": 1}]},
+                    }
+                ),
+            ):
+                with patch.object(async_client, "scroll", MockScroll()):
+
+                    data = [
+                        x
+                        async for x in helpers.async_scan(
+                            async_client,
+                            index="test_index",
+                            size=2,
+                            raise_on_error=False,
+                        )
+                    ]
+                    assert data == [{"search_data": 1}, {"scroll_data": 42}]
+
+            with patch.object(
+                async_client,
+                "search",
+                MockResponse(
+                    {
+                        "_scroll_id": "dummy_id",
+                        "_shards": {"successful": 4, "total": 5, "skipped": 0},
+                        "hits": {"hits": [{"search_data": 1}]},
+                    }
+                ),
+            ):
+                with patch.object(async_client, "scroll", MockScroll()) as mock_scroll:
+
+                    with pytest.raises(ScanError):
+                        data = [
+                            x
+                            async for x in helpers.async_scan(
+                                async_client,
+                                index="test_index",
+                                size=2,
+                                raise_on_error=True,
+                            )
+                        ]
+                        assert data == [{"search_data": 1}]
+                        assert mock_scroll.calls == []
+
+    async def test_no_scroll_id_fast_route(self, async_client, scan_teardown):
+        with patch.object(async_client, "search", MockResponse({"no": "_scroll_id"})):
+            with patch.object(async_client, "scroll") as scroll_mock:
+                with patch.object(async_client, "clear_scroll") as clear_mock:
+                    data = [
+                        x
+                        async for x in helpers.async_scan(
+                            async_client, index="test_index"
+                        )
+                    ]
+
+                    assert data == []
+                    scroll_mock.assert_not_called()
+                    clear_mock.assert_not_called()
+
+    @patch("opensearchpy._async.helpers.logger")
+    async def test_logger(self, logger_mock, async_client, scan_teardown):
+        bulk = []
+        for x in range(4):
+            bulk.append({"index": {"_index": "test_index"}})
+            bulk.append({"value": x})
+        await async_client.bulk(bulk, refresh=True)
+
+        with patch.object(async_client, "scroll", MockScroll()):
+            _ = [
+                x
+                async for x in helpers.async_scan(
+                    async_client,
+                    index="test_index",
+                    size=2,
+                    raise_on_error=False,
+                    clear_scroll=False,
+                )
+            ]
+            logger_mock.warning.assert_called()
+
+        with patch.object(async_client, "scroll", MockScroll()):
+            try:
+                _ = [
+                    x
+                    async for x in helpers.async_scan(
+                        async_client,
+                        index="test_index",
+                        size=2,
+                        raise_on_error=True,
+                        clear_scroll=False,
+                    )
+                ]
+            except ScanError:
+                pass
+            logger_mock.warning.assert_called_with(
+                "Scroll request has only succeeded on %d (+%d skipped) shards out of %d.",
+                4,
+                0,
+                5,
+            )
+
+    async def test_clear_scroll(self, async_client, scan_teardown):
+        bulk = []
+        for x in range(4):
+            bulk.append({"index": {"_index": "test_index"}})
+            bulk.append({"value": x})
+        await async_client.bulk(bulk, refresh=True)
+
+        with patch.object(
+            async_client, "clear_scroll", wraps=async_client.clear_scroll
+        ) as spy:
+            _ = [
+                x
+                async for x in helpers.async_scan(
+                    async_client, index="test_index", size=2
+                )
+            ]
+            spy.assert_called_once()
+
+            spy.reset_mock()
+            _ = [
+                x
+                async for x in helpers.async_scan(
+                    async_client, index="test_index", size=2, clear_scroll=True
+                )
+            ]
+            spy.assert_called_once()
+
+            spy.reset_mock()
+            _ = [
+                x
+                async for x in helpers.async_scan(
+                    async_client, index="test_index", size=2, clear_scroll=False
+                )
+            ]
+            spy.assert_not_called()
+
+    @pytest.mark.parametrize(
+        "kwargs",
+        [
+            {"api_key": ("name", "value")},
+            {"http_auth": ("username", "password")},
+            {"headers": {"custom", "header"}},
+        ],
+    )
+    async def test_scan_auth_kwargs_forwarded(
+        self, async_client, scan_teardown, kwargs
+    ):
+        ((key, val),) = kwargs.items()
+
+        with patch.object(
+            async_client,
+            "search",
+            return_value=MockResponse(
+                {
+                    "_scroll_id": "scroll_id",
+                    "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                    "hits": {"hits": [{"search_data": 1}]},
+                }
+            ),
+        ) as search_mock:
+            with patch.object(
+                async_client,
+                "scroll",
+                return_value=MockResponse(
+                    {
+                        "_scroll_id": "scroll_id",
+                        "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                        "hits": {"hits": []},
+                    }
+                ),
+            ) as scroll_mock:
+                with patch.object(
+                    async_client, "clear_scroll", return_value=MockResponse({})
+                ) as clear_mock:
+                    data = [
+                        x
+                        async for x in helpers.async_scan(
+                            async_client, index="test_index", **kwargs
+                        )
+                    ]
+
+                    assert data == [{"search_data": 1}]
+
+        for api_mock in (search_mock, scroll_mock, clear_mock):
+            assert api_mock.call_args[1][key] == val
+
+    async def test_scan_auth_kwargs_favor_scroll_kwargs_option(
+        self, async_client, scan_teardown
+    ):
+        with patch.object(
+            async_client,
+            "search",
+            return_value=MockResponse(
+                {
+                    "_scroll_id": "scroll_id",
+                    "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                    "hits": {"hits": [{"search_data": 1}]},
+                }
+            ),
+        ):
+            with patch.object(
+                async_client,
+                "scroll",
+                return_value=MockResponse(
+                    {
+                        "_scroll_id": "scroll_id",
+                        "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                        "hits": {"hits": []},
+                    }
+                ),
+            ):
+                with patch.object(
+                    async_client, "clear_scroll", return_value=MockResponse({})
+                ):
+                    data = [
+                        x
+                        async for x in helpers.async_scan(
+                            async_client,
+                            index="test_index",
+                            headers={"not scroll": "kwargs"},
+                            scroll_kwargs={
+                                "headers": {"scroll": "kwargs"},
+                                "sort": "asc",
+                            },
+                        )
+                    ]
+
+                    assert data == [{"search_data": 1}]
+
+                    # Assert that we see 'scroll_kwargs' options used instead of 'kwargs'
+                    assert async_client.scroll.call_args[1]["headers"] == {
+                        "scroll": "kwargs"
+                    }
+                    assert async_client.scroll.call_args[1]["sort"] == "asc"
+
+
+@pytest.fixture(scope="function")
+async def reindex_setup(async_client):
+    bulk = []
+    for x in range(100):
+        bulk.append({"index": {"_index": "test_index", "_id": x}})
+        bulk.append(
+            {
+                "answer": x,
+                "correct": x == 42,
+                "type": "answers" if x % 2 == 0 else "questions",
+            }
+        )
+    await async_client.bulk(bulk, refresh=True)
+    yield
+
+
+class TestReindex(object):
+    async def test_reindex_passes_kwargs_to_scan_and_bulk(
+        self, async_client, reindex_setup
+    ):
+        await helpers.async_reindex(
+            async_client,
+            "test_index",
+            "prod_index",
+            scan_kwargs={"q": "type:answers"},
+            bulk_kwargs={"refresh": True},
+        )
+
+        assert await async_client.indices.exists("prod_index")
+        assert (
+            50
+            == (await async_client.count(index="prod_index", q="type:answers"))["count"]
+        )
+
+        assert {"answer": 42, "correct": True, "type": "answers"} == (
+            await async_client.get(index="prod_index", id=42)
+        )["_source"]
+
+    async def test_reindex_accepts_a_query(self, async_client, reindex_setup):
+        await helpers.async_reindex(
+            async_client,
+            "test_index",
+            "prod_index",
+            query={"query": {"bool": {"filter": {"term": {"type": "answers"}}}}},
+        )
+        await async_client.indices.refresh()
+
+        assert await async_client.indices.exists("prod_index")
+        assert (
+            50
+            == (await async_client.count(index="prod_index", q="type:answers"))["count"]
+        )
+
+        assert {"answer": 42, "correct": True, "type": "answers"} == (
+            await async_client.get(index="prod_index", id=42)
+        )["_source"]
+
+    async def test_all_documents_get_moved(self, async_client, reindex_setup):
+        await helpers.async_reindex(async_client, "test_index", "prod_index")
+        await async_client.indices.refresh()
+
+        assert await async_client.indices.exists("prod_index")
+        assert (
+            50
+            == (await async_client.count(index="prod_index", q="type:questions"))[
+                "count"
+            ]
+        )
+        assert (
+            50
+            == (await async_client.count(index="prod_index", q="type:answers"))["count"]
+        )
+
+        assert {"answer": 42, "correct": True, "type": "answers"} == (
+            await async_client.get(index="prod_index", id=42)
+        )["_source"]
+
+
+@pytest.fixture(scope="function")
+async def parent_reindex_setup(async_client):
+    body = {
+        "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+        "mappings": {
+            "properties": {
+                "question_answer": {
+                    "type": "join",
+                    "relations": {"question": "answer"},
+                }
+            }
+        },
+    }
+    await async_client.indices.create(index="test-index", body=body)
+    await async_client.indices.create(index="real-index", body=body)
+
+    await async_client.index(
+        index="test-index", id=42, body={"question_answer": "question"}
+    )
+    await async_client.index(
+        index="test-index",
+        id=47,
+        routing=42,
+        body={"some": "data", "question_answer": {"name": "answer", "parent": 42}},
+    )
+    await async_client.indices.refresh(index="test-index")
+
+
+class TestParentChildReindex:
+    async def test_children_are_reindexed_correctly(
+        self, async_client, parent_reindex_setup
+    ):
+        await helpers.async_reindex(async_client, "test-index", "real-index")
+
+        q = await async_client.get(index="real-index", id=42)
+        assert {
+            "_id": "42",
+            "_index": "real-index",
+            "_primary_term": 1,
+            "_seq_no": 0,
+            "_source": {"question_answer": "question"},
+            "_type": "_doc",
+            "_version": 1,
+            "found": True,
+        } == q
+
+        q = await async_client.get(index="test-index", id=47, routing=42)
+        assert {
+            "_routing": "42",
+            "_id": "47",
+            "_index": "test-index",
+            "_primary_term": 1,
+            "_seq_no": 1,
+            "_source": {
+                "some": "data",
+                "question_answer": {"name": "answer", "parent": 42},
+            },
+            "_type": "_doc",
+            "_version": 1,
+            "found": True,
+        } == q

test_opensearchpy/test_async/test_server/test_rest_api_spec.py

@@ -0,0 +1,233 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+"""
+Dynamically generated set of TestCases based on set of yaml files decribing
+some integration tests. These files are shared among all official OpenSearch
+clients.
+"""
+import inspect
+import warnings
+
+import pytest
+
+from opensearchpy import OpenSearchWarning
+from opensearchpy.helpers.test import _get_version
+
+from ...test_server.test_rest_api_spec import (
+    IMPLEMENTED_FEATURES,
+    PARAMS_RENAMES,
+    RUN_ASYNC_REST_API_TESTS,
+    YAML_TEST_SPECS,
+    YamlRunner,
+)
+
+pytestmark = pytest.mark.asyncio
+
+OPENSEARCH_VERSION = None
+
+
+async def await_if_coro(x):
+    if inspect.iscoroutine(x):
+        return await x
+    return x
+
+
+class AsyncYamlRunner(YamlRunner):
+    async def setup(self):
+        # Pull skips from individual tests to not do unnecessary setup.
+        skip_code = []
+        for action in self._run_code:
+            assert len(action) == 1
+            action_type, _ = list(action.items())[0]
+            if action_type == "skip":
+                skip_code.append(action)
+            else:
+                break
+
+        if self._setup_code or skip_code:
+            self.section("setup")
+        if skip_code:
+            await self.run_code(skip_code)
+        if self._setup_code:
+            await self.run_code(self._setup_code)
+
+    async def teardown(self):
+        if self._teardown_code:
+            self.section("teardown")
+            await self.run_code(self._teardown_code)
+
+    async def opensearch_version(self):
+        global OPENSEARCH_VERSION
+        if OPENSEARCH_VERSION is None:
+            version_string = (await self.client.info())["version"]["number"]
+            if "." not in version_string:
+                return ()
+            version = version_string.strip().split(".")
+            OPENSEARCH_VERSION = tuple(int(v) if v.isdigit() else 999 for v in version)
+        return OPENSEARCH_VERSION
+
+    def section(self, name):
+        print(("=" * 10) + " " + name + " " + ("=" * 10))
+
+    async def run(self):
+        try:
+            await self.setup()
+            self.section("test")
+            await self.run_code(self._run_code)
+        finally:
+            try:
+                await self.teardown()
+            except Exception:
+                pass
+
+    async def run_code(self, test):
+        """Execute an instruction based on it's type."""
+        for action in test:
+            assert len(action) == 1
+            action_type, action = list(action.items())[0]
+            print(action_type, action)
+
+            if hasattr(self, "run_" + action_type):
+                await await_if_coro(getattr(self, "run_" + action_type)(action))
+            else:
+                raise RuntimeError("Invalid action type %r" % (action_type,))
+
+    async def run_do(self, action):
+        api = self.client
+        headers = action.pop("headers", None)
+        catch = action.pop("catch", None)
+        warn = action.pop("warnings", ())
+        allowed_warnings = action.pop("allowed_warnings", ())
+        assert len(action) == 1
+
+        # Remove the x_pack_rest_user authentication
+        # if it's given via headers. We're already authenticated
+        # via the 'elastic' user.
+        if (
+            headers
+            and headers.get("Authorization", None)
+            == "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA=="
+        ):
+            headers.pop("Authorization")
+
+        method, args = list(action.items())[0]
+        args["headers"] = headers
+
+        # locate api endpoint
+        for m in method.split("."):
+            assert hasattr(api, m)
+            api = getattr(api, m)
+
+        # some parameters had to be renamed to not clash with python builtins,
+        # compensate
+        for k in PARAMS_RENAMES:
+            if k in args:
+                args[PARAMS_RENAMES[k]] = args.pop(k)
+
+        # resolve vars
+        for k in args:
+            args[k] = self._resolve(args[k])
+
+        warnings.simplefilter("always", category=OpenSearchWarning)
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            try:
+                self.last_response = await api(**args)
+            except Exception as e:
+                if not catch:
+                    raise
+                self.run_catch(catch, e)
+            else:
+                if catch:
+                    raise AssertionError(
+                        "Failed to catch %r in %r." % (catch, self.last_response)
+                    )
+
+        # Filter out warnings raised by other components.
+        caught_warnings = [
+            str(w.message)
+            for w in caught_warnings
+            if w.category == OpenSearchWarning
+            and str(w.message) not in allowed_warnings
+        ]
+
+        # This warning can show up in many places but isn't accounted for
+        # in tests, so we remove it to make sure things pass.
+        include_type_name_warning = (
+            "[types removal] Using include_type_name in create index requests is deprecated. "
+            "The parameter will be removed in the next major version."
+        )
+        if (
+            include_type_name_warning in caught_warnings
+            and include_type_name_warning not in warn
+        ):
+            caught_warnings.remove(include_type_name_warning)
+
+        # Sorting removes the issue with order raised. We only care about
+        # if all warnings are raised in the single API call.
+        if warn and sorted(warn) != sorted(caught_warnings):
+            raise AssertionError(
+                "Expected warnings not equal to actual warnings: expected=%r actual=%r"
+                % (warn, caught_warnings)
+            )
+
+    async def run_skip(self, skip):
+        if "features" in skip:
+            features = skip["features"]
+            if not isinstance(features, (tuple, list)):
+                features = [features]
+            for feature in features:
+                if feature in IMPLEMENTED_FEATURES:
+                    continue
+                pytest.skip("feature '%s' is not supported" % feature)
+
+        if "version" in skip:
+            version, reason = skip["version"], skip["reason"]
+            if version == "all":
+                pytest.skip(reason)
+            min_version, max_version = version.split("-")
+            min_version = _get_version(min_version) or (0,)
+            max_version = _get_version(max_version) or (999,)
+            if min_version <= (await self.opensearch_version()) <= max_version:
+                pytest.skip(reason)
+
+    async def _feature_enabled(self, name):
+        return False
+
+
+@pytest.fixture(scope="function")
+def async_runner(async_client):
+    return AsyncYamlRunner(async_client)
+
+
+if RUN_ASYNC_REST_API_TESTS:
+
+    @pytest.mark.parametrize("test_spec", YAML_TEST_SPECS)
+    async def test_rest_api_spec(test_spec, async_runner):
+        if test_spec.get("skip", False):
+            pytest.skip("Manually skipped in 'SKIP_TESTS'")
+        async_runner.use_spec(test_spec)
+        await async_runner.run()

test_opensearchpy/test_async/test_transport.py

@@ -0,0 +1,549 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from __future__ import unicode_literals
+
+import asyncio
+import json
+
+import pytest
+from mock import patch
+
+from opensearchpy import AsyncTransport
+from opensearchpy.connection import Connection
+from opensearchpy.connection_pool import DummyConnectionPool
+from opensearchpy.exceptions import ConnectionError, TransportError
+
+pytestmark = pytest.mark.asyncio
+
+
+class DummyConnection(Connection):
+    def __init__(self, **kwargs):
+        self.exception = kwargs.pop("exception", None)
+        self.status, self.data = kwargs.pop("status", 200), kwargs.pop("data", "{}")
+        self.headers = kwargs.pop("headers", {})
+        self.delay = kwargs.pop("delay", 0)
+        self.calls = []
+        self.closed = False
+        super(DummyConnection, self).__init__(**kwargs)
+
+    async def perform_request(self, *args, **kwargs):
+        if self.closed:
+            raise RuntimeError("This connection is closed")
+        if self.delay:
+            await asyncio.sleep(self.delay)
+        self.calls.append((args, kwargs))
+        if self.exception:
+            raise self.exception
+        return self.status, self.headers, self.data
+
+    async def close(self):
+        if self.closed:
+            raise RuntimeError("This connection is already closed")
+        self.closed = True
+
+
+CLUSTER_NODES = """{
+  "_nodes" : {
+    "total" : 1,
+    "successful" : 1,
+    "failed" : 0
+  },
+  "cluster_name" : "opensearch",
+  "nodes" : {
+    "SRZpKFZdQguhhvifmN6UVA" : {
+      "name" : "SRZpKFZ",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1",
+      "version" : "5.0.0",
+      "build_hash" : "253032b",
+      "roles" : [ "master", "data", "ingest" ],
+      "http" : {
+        "bound_address" : [ "[fe80::1]:9200", "[::1]:9200", "127.0.0.1:9200" ],
+        "publish_address" : "1.1.1.1:123",
+        "max_content_length_in_bytes" : 104857600
+      }
+    }
+  }
+}"""
+
+CLUSTER_NODES_7x_PUBLISH_HOST = """{
+  "_nodes" : {
+    "total" : 1,
+    "successful" : 1,
+    "failed" : 0
+  },
+  "cluster_name" : "opensearch",
+  "nodes" : {
+    "SRZpKFZdQguhhvifmN6UVA" : {
+      "name" : "SRZpKFZ",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1",
+      "version" : "5.0.0",
+      "build_hash" : "253032b",
+      "roles" : [ "master", "data", "ingest" ],
+      "http" : {
+        "bound_address" : [ "[fe80::1]:9200", "[::1]:9200", "127.0.0.1:9200" ],
+        "publish_address" : "somehost.tld/1.1.1.1:123",
+        "max_content_length_in_bytes" : 104857600
+      }
+    }
+  }
+}"""
+
+
+class TestTransport:
+    async def test_single_connection_uses_dummy_connection_pool(self):
+        t = AsyncTransport([{}])
+        await t._async_call()
+        assert isinstance(t.connection_pool, DummyConnectionPool)
+        t = AsyncTransport([{"host": "localhost"}])
+        await t._async_call()
+        assert isinstance(t.connection_pool, DummyConnectionPool)
+
+    async def test_request_timeout_extracted_from_params_and_passed(self):
+        t = AsyncTransport([{}], connection_class=DummyConnection)
+
+        await t.perform_request("GET", "/", params={"request_timeout": 42})
+        assert 1 == len(t.get_connection().calls)
+        assert ("GET", "/", {}, None) == t.get_connection().calls[0][0]
+        assert {
+            "timeout": 42,
+            "ignore": (),
+            "headers": None,
+        } == t.get_connection().calls[0][1]
+
+    async def test_opaque_id(self):
+        t = AsyncTransport([{}], opaque_id="app-1", connection_class=DummyConnection)
+
+        await t.perform_request("GET", "/")
+        assert 1 == len(t.get_connection().calls)
+        assert ("GET", "/", None, None) == t.get_connection().calls[0][0]
+        assert {
+            "timeout": None,
+            "ignore": (),
+            "headers": None,
+        } == t.get_connection().calls[0][1]
+
+        # Now try with an 'x-opaque-id' set on perform_request().
+        await t.perform_request("GET", "/", headers={"x-opaque-id": "request-1"})
+        assert 2 == len(t.get_connection().calls)
+        assert ("GET", "/", None, None) == t.get_connection().calls[1][0]
+        assert {
+            "timeout": None,
+            "ignore": (),
+            "headers": {"x-opaque-id": "request-1"},
+        } == t.get_connection().calls[1][1]
+
+    async def test_request_with_custom_user_agent_header(self):
+        t = AsyncTransport([{}], connection_class=DummyConnection)
+
+        await t.perform_request(
+            "GET", "/", headers={"user-agent": "my-custom-value/1.2.3"}
+        )
+        assert 1 == len(t.get_connection().calls)
+        assert {
+            "timeout": None,
+            "ignore": (),
+            "headers": {"user-agent": "my-custom-value/1.2.3"},
+        } == t.get_connection().calls[0][1]
+
+    async def test_send_get_body_as_source(self):
+        t = AsyncTransport(
+            [{}], send_get_body_as="source", connection_class=DummyConnection
+        )
+
+        await t.perform_request("GET", "/", body={})
+        assert 1 == len(t.get_connection().calls)
+        assert ("GET", "/", {"source": "{}"}, None) == t.get_connection().calls[0][0]
+
+    async def test_send_get_body_as_post(self):
+        t = AsyncTransport(
+            [{}], send_get_body_as="POST", connection_class=DummyConnection
+        )
+
+        await t.perform_request("GET", "/", body={})
+        assert 1 == len(t.get_connection().calls)
+        assert ("POST", "/", None, b"{}") == t.get_connection().calls[0][0]
+
+    async def test_body_gets_encoded_into_bytes(self):
+        t = AsyncTransport([{}], connection_class=DummyConnection)
+
+        await t.perform_request("GET", "/", body="你好")
+        assert 1 == len(t.get_connection().calls)
+        assert (
+            "GET",
+            "/",
+            None,
+            b"\xe4\xbd\xa0\xe5\xa5\xbd",
+        ) == t.get_connection().calls[0][0]
+
+    async def test_body_bytes_get_passed_untouched(self):
+        t = AsyncTransport([{}], connection_class=DummyConnection)
+
+        body = b"\xe4\xbd\xa0\xe5\xa5\xbd"
+        await t.perform_request("GET", "/", body=body)
+        assert 1 == len(t.get_connection().calls)
+        assert ("GET", "/", None, body) == t.get_connection().calls[0][0]
+
+    async def test_body_surrogates_replaced_encoded_into_bytes(self):
+        t = AsyncTransport([{}], connection_class=DummyConnection)
+
+        await t.perform_request("GET", "/", body="你好\uda6a")
+        assert 1 == len(t.get_connection().calls)
+        assert (
+            "GET",
+            "/",
+            None,
+            b"\xe4\xbd\xa0\xe5\xa5\xbd\xed\xa9\xaa",
+        ) == t.get_connection().calls[0][0]
+
+    async def test_kwargs_passed_on_to_connections(self):
+        t = AsyncTransport([{"host": "google.com"}], port=123)
+        await t._async_call()
+        assert 1 == len(t.connection_pool.connections)
+        assert "http://google.com:123" == t.connection_pool.connections[0].host
+
+    async def test_kwargs_passed_on_to_connection_pool(self):
+        dt = object()
+        t = AsyncTransport([{}, {}], dead_timeout=dt)
+        await t._async_call()
+        assert dt is t.connection_pool.dead_timeout
+
+    async def test_custom_connection_class(self):
+        class MyConnection(object):
+            def __init__(self, **kwargs):
+                self.kwargs = kwargs
+
+        t = AsyncTransport([{}], connection_class=MyConnection)
+        await t._async_call()
+        assert 1 == len(t.connection_pool.connections)
+        assert isinstance(t.connection_pool.connections[0], MyConnection)
+
+    def test_add_connection(self):
+        t = AsyncTransport([{}], randomize_hosts=False)
+        t.add_connection({"host": "google.com", "port": 1234})
+
+        assert 2 == len(t.connection_pool.connections)
+        assert "http://google.com:1234" == t.connection_pool.connections[1].host
+
+    async def test_request_will_fail_after_X_retries(self):
+        t = AsyncTransport(
+            [{"exception": ConnectionError("abandon ship")}],
+            connection_class=DummyConnection,
+        )
+
+        connection_error = False
+        try:
+            await t.perform_request("GET", "/")
+        except ConnectionError:
+            connection_error = True
+
+        assert connection_error
+        assert 4 == len(t.get_connection().calls)
+
+    async def test_failed_connection_will_be_marked_as_dead(self):
+        t = AsyncTransport(
+            [{"exception": ConnectionError("abandon ship")}] * 2,
+            connection_class=DummyConnection,
+        )
+
+        connection_error = False
+        try:
+            await t.perform_request("GET", "/")
+        except ConnectionError:
+            connection_error = True
+
+        assert connection_error
+        assert 0 == len(t.connection_pool.connections)
+
+    async def test_resurrected_connection_will_be_marked_as_live_on_success(self):
+        for method in ("GET", "HEAD"):
+            t = AsyncTransport([{}, {}], connection_class=DummyConnection)
+            await t._async_call()
+            con1 = t.connection_pool.get_connection()
+            con2 = t.connection_pool.get_connection()
+            t.connection_pool.mark_dead(con1)
+            t.connection_pool.mark_dead(con2)
+
+            await t.perform_request(method, "/")
+            assert 1 == len(t.connection_pool.connections)
+            assert 1 == len(t.connection_pool.dead_count)
+
+    async def test_sniff_will_use_seed_connections(self):
+        t = AsyncTransport([{"data": CLUSTER_NODES}], connection_class=DummyConnection)
+        await t._async_call()
+        t.set_connections([{"data": "invalid"}])
+
+        await t.sniff_hosts()
+        assert 1 == len(t.connection_pool.connections)
+        assert "http://1.1.1.1:123" == t.get_connection().host
+
+    async def test_sniff_on_start_fetches_and_uses_nodes_list(self):
+        t = AsyncTransport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_start=True,
+        )
+        await t._async_call()
+        await t.sniffing_task  # Need to wait for the sniffing task to complete
+
+        assert 1 == len(t.connection_pool.connections)
+        assert "http://1.1.1.1:123" == t.get_connection().host
+
+    async def test_sniff_on_start_ignores_sniff_timeout(self):
+        t = AsyncTransport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_start=True,
+            sniff_timeout=12,
+        )
+        await t._async_call()
+        await t.sniffing_task  # Need to wait for the sniffing task to complete
+
+        assert (("GET", "/_nodes/_all/http"), {"timeout": None}) == t.seed_connections[
+            0
+        ].calls[0]
+
+    async def test_sniff_uses_sniff_timeout(self):
+        t = AsyncTransport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_timeout=42,
+        )
+        await t._async_call()
+        await t.sniff_hosts()
+
+        assert (("GET", "/_nodes/_all/http"), {"timeout": 42}) == t.seed_connections[
+            0
+        ].calls[0]
+
+    async def test_sniff_reuses_connection_instances_if_possible(self):
+        t = AsyncTransport(
+            [{"data": CLUSTER_NODES}, {"host": "1.1.1.1", "port": 123}],
+            connection_class=DummyConnection,
+            randomize_hosts=False,
+        )
+        await t._async_call()
+        connection = t.connection_pool.connections[1]
+        connection.delay = 3.0  # Add this delay to make the sniffing deterministic.
+
+        await t.sniff_hosts()
+        assert 1 == len(t.connection_pool.connections)
+        assert connection is t.get_connection()
+
+    async def test_sniff_on_fail_triggers_sniffing_on_fail(self):
+        t = AsyncTransport(
+            [{"exception": ConnectionError("abandon ship")}, {"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_connection_fail=True,
+            max_retries=0,
+            randomize_hosts=False,
+        )
+        await t._async_call()
+
+        connection_error = False
+        try:
+            await t.perform_request("GET", "/")
+        except ConnectionError:
+            connection_error = True
+
+        await t.sniffing_task  # Need to wait for the sniffing task to complete
+
+        assert connection_error
+        assert 1 == len(t.connection_pool.connections)
+        assert "http://1.1.1.1:123" == t.get_connection().host
+
+    @patch("opensearchpy._async.transport.AsyncTransport.sniff_hosts")
+    async def test_sniff_on_fail_failing_does_not_prevent_retires(self, sniff_hosts):
+        sniff_hosts.side_effect = [TransportError("sniff failed")]
+        t = AsyncTransport(
+            [{"exception": ConnectionError("abandon ship")}, {"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_connection_fail=True,
+            max_retries=3,
+            randomize_hosts=False,
+        )
+        await t._async_init()
+
+        conn_err, conn_data = t.connection_pool.connections
+        response = await t.perform_request("GET", "/")
+        assert json.loads(CLUSTER_NODES) == response
+        assert 1 == sniff_hosts.call_count
+        assert 1 == len(conn_err.calls)
+        assert 1 == len(conn_data.calls)
+
+    async def test_sniff_after_n_seconds(self, event_loop):
+        t = AsyncTransport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniffer_timeout=5,
+        )
+        await t._async_call()
+
+        for _ in range(4):
+            await t.perform_request("GET", "/")
+        assert 1 == len(t.connection_pool.connections)
+        assert isinstance(t.get_connection(), DummyConnection)
+        t.last_sniff = event_loop.time() - 5.1
+
+        await t.perform_request("GET", "/")
+        await t.sniffing_task  # Need to wait for the sniffing task to complete
+
+        assert 1 == len(t.connection_pool.connections)
+        assert "http://1.1.1.1:123" == t.get_connection().host
+        assert event_loop.time() - 1 < t.last_sniff < event_loop.time() + 0.01
+
+    async def test_sniff_7x_publish_host(self):
+        # Test the response shaped when a 7.x node has publish_host set
+        # and the returend data is shaped in the fqdn/ip:port format.
+        t = AsyncTransport(
+            [{"data": CLUSTER_NODES_7x_PUBLISH_HOST}],
+            connection_class=DummyConnection,
+            sniff_timeout=42,
+        )
+        await t._async_call()
+        await t.sniff_hosts()
+        # Ensure we parsed out the fqdn and port from the fqdn/ip:port string.
+        assert t.connection_pool.connection_opts[0][1] == {
+            "host": "somehost.tld",
+            "port": 123,
+        }
+
+    @patch("opensearchpy._async.transport.AsyncTransport.sniff_hosts")
+    async def test_sniffing_disabled_on_cloud_instances(self, sniff_hosts):
+        t = AsyncTransport(
+            [{}],
+            sniff_on_start=True,
+            sniff_on_connection_fail=True,
+            connection_class=DummyConnection,
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+        )
+        await t._async_call()
+
+        assert not t.sniff_on_connection_fail
+        assert sniff_hosts.call_args is None  # Assert not called.
+        await t.perform_request("GET", "/", body={})
+        assert 1 == len(t.get_connection().calls)
+        assert ("GET", "/", None, b"{}") == t.get_connection().calls[0][0]
+
+    async def test_transport_close_closes_all_pool_connections(self):
+        t = AsyncTransport([{}], connection_class=DummyConnection)
+        await t._async_call()
+
+        assert not any([conn.closed for conn in t.connection_pool.connections])
+        await t.close()
+        assert all([conn.closed for conn in t.connection_pool.connections])
+
+        t = AsyncTransport([{}, {}], connection_class=DummyConnection)
+        await t._async_call()
+
+        assert not any([conn.closed for conn in t.connection_pool.connections])
+        await t.close()
+        assert all([conn.closed for conn in t.connection_pool.connections])
+
+    async def test_sniff_on_start_error_if_no_sniffed_hosts(self, event_loop):
+        t = AsyncTransport(
+            [
+                {"data": ""},
+                {"data": ""},
+                {"data": ""},
+            ],
+            connection_class=DummyConnection,
+            sniff_on_start=True,
+        )
+
+        # If our initial sniffing attempt comes back
+        # empty then we raise an error.
+        with pytest.raises(TransportError) as e:
+            await t._async_call()
+        assert str(e.value) == "TransportError(N/A, 'Unable to sniff hosts.')"
+
+    async def test_sniff_on_start_waits_for_sniff_to_complete(self, event_loop):
+        t = AsyncTransport(
+            [
+                {"delay": 1, "data": ""},
+                {"delay": 1, "data": ""},
+                {"delay": 1, "data": CLUSTER_NODES},
+            ],
+            connection_class=DummyConnection,
+            sniff_on_start=True,
+        )
+
+        # Start the timer right before the first task
+        # and have a bunch of tasks come in immediately.
+        tasks = []
+        start_time = event_loop.time()
+        for _ in range(5):
+            tasks.append(event_loop.create_task(t._async_call()))
+            await asyncio.sleep(0)  # Yield to the loop
+
+        assert t.sniffing_task is not None
+
+        # Tasks streaming in later.
+        for _ in range(5):
+            tasks.append(event_loop.create_task(t._async_call()))
+            await asyncio.sleep(0.1)
+
+        # Now that all the API calls have come in we wait for
+        # them all to resolve before
+        await asyncio.gather(*tasks)
+        end_time = event_loop.time()
+        duration = end_time - start_time
+
+        # All the tasks blocked on the sniff of each node
+        # and then resolved immediately after.
+        assert 1 <= duration < 2
+
+    async def test_sniff_on_start_close_unlocks_async_calls(self, event_loop):
+        t = AsyncTransport(
+            [
+                {"delay": 10, "data": CLUSTER_NODES},
+            ],
+            connection_class=DummyConnection,
+            sniff_on_start=True,
+        )
+
+        # Start making _async_calls() before we cancel
+        tasks = []
+        start_time = event_loop.time()
+        for _ in range(3):
+            tasks.append(event_loop.create_task(t._async_call()))
+            await asyncio.sleep(0)
+
+        # Close the transport while the sniffing task is active! :(
+        await t.close()
+
+        # Now we start waiting on all those _async_calls()
+        await asyncio.gather(*tasks)
+        end_time = event_loop.time()
+        duration = end_time - start_time
+
+        # A lot quicker than 10 seconds defined in 'delay'
+        assert duration < 1

test_opensearchpy/test_cases.py

@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from collections import defaultdict
+from unittest import SkipTest  # noqa: F401
+from unittest import TestCase
+
+from opensearchpy import OpenSearch
+
+
+class DummyTransport(object):
+    def __init__(self, hosts, responses=None, **kwargs):
+        self.hosts = hosts
+        self.responses = responses
+        self.call_count = 0
+        self.calls = defaultdict(list)
+
+    def perform_request(self, method, url, params=None, headers=None, body=None):
+        resp = 200, {}
+        if self.responses:
+            resp = self.responses[self.call_count]
+        self.call_count += 1
+        self.calls[(method, url)].append((params, headers, body))
+        return resp
+
+
+class OpenSearchTestCase(TestCase):
+    def setUp(self):
+        super(OpenSearchTestCase, self).setUp()
+        self.client = OpenSearch(transport_class=DummyTransport)
+
+    def assert_call_count_equals(self, count):
+        self.assertEqual(count, self.client.transport.call_count)
+
+    def assert_url_called(self, method, url, count=1):
+        self.assertIn((method, url), self.client.transport.calls)
+        calls = self.client.transport.calls[(method, url)]
+        self.assertEqual(count, len(calls))
+        return calls
+
+
+class TestOpenSearchTestCase(OpenSearchTestCase):
+    def test_our_transport_used(self):
+        self.assertIsInstance(self.client.transport, DummyTransport)
+
+    def test_start_with_0_call(self):
+        self.assert_call_count_equals(0)
+
+    def test_each_call_is_recorded(self):
+        self.client.transport.perform_request("GET", "/")
+        self.client.transport.perform_request("DELETE", "/42", params={}, body="body")
+        self.assert_call_count_equals(2)
+        self.assertEqual(
+            [({}, None, "body")], self.assert_url_called("DELETE", "/42", 1)
+        )

test_opensearchpy/test_client/__init__.py

@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from __future__ import unicode_literals
+
+import warnings
+
+from opensearchpy.client import OpenSearch, _normalize_hosts
+
+from ..test_cases import OpenSearchTestCase, TestCase
+
+
+class TestNormalizeHosts(TestCase):
+    def test_none_uses_defaults(self):
+        self.assertEqual([{}], _normalize_hosts(None))
+
+    def test_strings_are_used_as_hostnames(self):
+        self.assertEqual([{"host": "elastic.co"}], _normalize_hosts(["elastic.co"]))
+
+    def test_strings_are_parsed_for_port_and_user(self):
+        self.assertEqual(
+            [
+                {"host": "elastic.co", "port": 42},
+                {"host": "elastic.co", "http_auth": "user:secre]"},
+            ],
+            _normalize_hosts(["elastic.co:42", "user:secre%[email protected]"]),
+        )
+
+    def test_strings_are_parsed_for_scheme(self):
+        self.assertEqual(
+            [
+                {"host": "elastic.co", "port": 42, "use_ssl": True},
+                {
+                    "host": "elastic.co",
+                    "http_auth": "user:secret",
+                    "use_ssl": True,
+                    "port": 443,
+                    "url_prefix": "/prefix",
+                },
+            ],
+            _normalize_hosts(
+                ["https://elastic.co:42", "https://user:[email protected]/prefix"]
+            ),
+        )
+
+    def test_dicts_are_left_unchanged(self):
+        self.assertEqual(
+            [{"host": "local", "extra": 123}],
+            _normalize_hosts([{"host": "local", "extra": 123}]),
+        )
+
+    def test_single_string_is_wrapped_in_list(self):
+        self.assertEqual([{"host": "elastic.co"}], _normalize_hosts("elastic.co"))
+
+
+class TestClient(OpenSearchTestCase):
+    def test_request_timeout_is_passed_through_unescaped(self):
+        self.client.ping(request_timeout=0.1)
+        calls = self.assert_url_called("HEAD", "/")
+        self.assertEqual([({"request_timeout": 0.1}, {}, None)], calls)
+
+    def test_params_is_copied_when(self):
+        rt = object()
+        params = dict(request_timeout=rt)
+        self.client.ping(params=params)
+        self.client.ping(params=params)
+        calls = self.assert_url_called("HEAD", "/", 2)
+        self.assertEqual(
+            [({"request_timeout": rt}, {}, None), ({"request_timeout": rt}, {}, None)],
+            calls,
+        )
+        self.assertFalse(calls[0][0] is calls[1][0])
+
+    def test_headers_is_copied_when(self):
+        hv = "value"
+        headers = dict(Authentication=hv)
+        self.client.ping(headers=headers)
+        self.client.ping(headers=headers)
+        calls = self.assert_url_called("HEAD", "/", 2)
+        self.assertEqual(
+            [({}, {"authentication": hv}, None), ({}, {"authentication": hv}, None)],
+            calls,
+        )
+        self.assertFalse(calls[0][0] is calls[1][0])
+
+    def test_from_in_search(self):
+        self.client.search(index="i", from_=10)
+        calls = self.assert_url_called("POST", "/i/_search")
+        self.assertEqual([({"from": "10"}, {}, None)], calls)
+
+    def test_repr_contains_hosts(self):
+        self.assertEqual("<OpenSearch([{}])>", repr(self.client))
+
+    def test_repr_subclass(self):
+        class OtherOpenSearch(OpenSearch):
+            pass
+
+        self.assertEqual("<OtherOpenSearch([{}])>", repr(OtherOpenSearch()))
+
+    def test_repr_contains_hosts_passed_in(self):
+        self.assertIn("opensearchpy.org", repr(OpenSearch(["opensearch.org:123"])))
+
+    def test_repr_truncates_host_to_5(self):
+        hosts = [{"host": "opensearch" + str(i)} for i in range(10)]
+        client = OpenSearch(hosts)
+        self.assertNotIn("opensearch5", repr(client))
+        self.assertIn("...", repr(client))
+
+    def test_index_uses_post_if_id_is_empty(self):
+        self.client.index(index="my-index", id="", body={})
+
+        self.assert_url_called("POST", "/my-index/_doc")
+
+    def test_index_uses_put_if_id_is_not_empty(self):
+        self.client.index(index="my-index", id=0, body={})
+
+        self.assert_url_called("PUT", "/my-index/_doc/0")
+
+    def test_tasks_get_without_task_id_deprecated(self):
+        warnings.simplefilter("always", DeprecationWarning)
+        with warnings.catch_warnings(record=True) as w:
+            self.client.tasks.get()
+
+        self.assert_url_called("GET", "/_tasks")
+        self.assertEqual(len(w), 1)
+        self.assertIs(w[0].category, DeprecationWarning)
+        self.assertEqual(
+            str(w[0].message),
+            "Calling client.tasks.get() without a task_id is deprecated "
+            "and will be removed in v8.0. Use client.tasks.list() instead.",
+        )
+
+    def test_tasks_get_with_task_id_not_deprecated(self):
+        warnings.simplefilter("always", DeprecationWarning)
+        with warnings.catch_warnings(record=True) as w:
+            self.client.tasks.get("task-1")
+            self.client.tasks.get(task_id="task-2")
+
+        self.assert_url_called("GET", "/_tasks/task-1")
+        self.assert_url_called("GET", "/_tasks/task-2")
+        self.assertEqual(len(w), 0)

test_opensearchpy/test_client/test_cluster.py

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from test_opensearchpy.test_cases import OpenSearchTestCase
+
+
+class TestCluster(OpenSearchTestCase):
+    def test_stats_without_node_id(self):
+        self.client.cluster.stats()
+        self.assert_url_called("GET", "/_cluster/stats")
+
+    def test_stats_with_node_id(self):
+        self.client.cluster.stats("node-1")
+        self.assert_url_called("GET", "/_cluster/stats/nodes/node-1")
+
+        self.client.cluster.stats(node_id="node-2")
+        self.assert_url_called("GET", "/_cluster/stats/nodes/node-2")
+
+    def test_state_with_index_without_metric_defaults_to_all(self):
+        self.client.cluster.state()
+        self.assert_url_called("GET", "/_cluster/state")
+
+        self.client.cluster.state(metric="cluster_name")
+        self.assert_url_called("GET", "/_cluster/state/cluster_name")
+
+        self.client.cluster.state(index="index-1")
+        self.assert_url_called("GET", "/_cluster/state/_all/index-1")
+
+        self.client.cluster.state(index="index-1", metric="cluster_name")
+        self.assert_url_called("GET", "/_cluster/state/cluster_name/index-1")

test_opensearchpy/test_client/test_indices.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from test_opensearchpy.test_cases import OpenSearchTestCase
+
+
+class TestIndices(OpenSearchTestCase):
+    def test_create_one_index(self):
+        self.client.indices.create("test-index")
+        self.assert_url_called("PUT", "/test-index")
+
+    def test_delete_multiple_indices(self):
+        self.client.indices.delete(["test-index", "second.index", "third/index"])
+        self.assert_url_called("DELETE", "/test-index,second.index,third%2Findex")
+
+    def test_exists_index(self):
+        self.client.indices.exists("second.index,third/index")
+        self.assert_url_called("HEAD", "/second.index,third%2Findex")
+
+    def test_passing_empty_value_for_required_param_raises_exception(self):
+        self.assertRaises(ValueError, self.client.indices.exists, index=None)
+        self.assertRaises(ValueError, self.client.indices.exists, index=[])
+        self.assertRaises(ValueError, self.client.indices.exists, index="")
+
+    def test_put_mapping_without_index(self):
+        self.client.indices.put_mapping(doc_type="doc-type", body={})
+        self.assert_url_called("PUT", "/_all/doc-type/_mapping")

test_opensearchpy/test_client/test_overrides.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import pytest
+
+from test_opensearchpy.test_cases import OpenSearchTestCase
+
+
+class TestOverriddenUrlTargets(OpenSearchTestCase):
+    def test_create(self):
+        self.client.create(index="test-index", id="test-id", body={})
+        self.assert_url_called("PUT", "/test-index/_create/test-id")
+
+        self.client.create(
+            index="test-index", doc_type="test-type", id="test-id", body={}
+        )
+        self.assert_url_called("PUT", "/test-index/test-type/test-id/_create")
+
+    def test_delete(self):
+        self.client.delete(index="test-index", id="test-id")
+        self.assert_url_called("DELETE", "/test-index/_doc/test-id")
+
+        self.client.delete(index="test-index", doc_type="test-type", id="test-id")
+        self.assert_url_called("DELETE", "/test-index/test-type/test-id")
+
+    def test_exists(self):
+        self.client.exists(index="test-index", id="test-id")
+        self.assert_url_called("HEAD", "/test-index/_doc/test-id")
+
+        self.client.exists(index="test-index", doc_type="test-type", id="test-id")
+        self.assert_url_called("HEAD", "/test-index/test-type/test-id")
+
+    def test_explain(self):
+        self.client.explain(index="test-index", id="test-id")
+        self.assert_url_called("POST", "/test-index/_explain/test-id")
+
+        self.client.explain(index="test-index", doc_type="test-type", id="test-id")
+        self.assert_url_called("POST", "/test-index/test-type/test-id/_explain")
+
+    def test_get(self):
+        self.client.get(index="test-index", id="test-id")
+        self.assert_url_called("GET", "/test-index/_doc/test-id")
+
+        self.client.get(index="test-index", doc_type="test-type", id="test-id")
+        self.assert_url_called("GET", "/test-index/test-type/test-id")
+
+    def test_get_source(self):
+        self.client.get_source(index="test-index", id="test-id")
+        self.assert_url_called("GET", "/test-index/_source/test-id")
+
+        self.client.get_source(index="test-index", doc_type="test-type", id="test-id")
+        self.assert_url_called("GET", "/test-index/test-type/test-id/_source")
+
+    def test_exists_source(self):
+        self.client.exists_source(index="test-index", id="test-id")
+        self.assert_url_called("HEAD", "/test-index/_source/test-id")
+
+        self.client.exists_source(
+            index="test-index", doc_type="test-type", id="test-id"
+        )
+        self.assert_url_called("HEAD", "/test-index/test-type/test-id/_source")
+
+    def test_index(self):
+        self.client.index(index="test-index", body={})
+        self.assert_url_called("POST", "/test-index/_doc")
+
+        self.client.index(index="test-index", id="test-id", body={})
+        self.assert_url_called("PUT", "/test-index/_doc/test-id")
+
+        self.client.index(index="test-index", doc_type="test-type", body={})
+        self.assert_url_called("POST", "/test-index/test-type")
+
+        self.client.index(
+            index="test-index", doc_type="test-type", id="test-id", body={}
+        )
+        self.assert_url_called("PUT", "/test-index/test-type/test-id")
+
+        self.client.index(index="test-index", doc_type="_doc", body={})
+        self.assert_url_called("POST", "/test-index/_doc", count=2)
+
+        self.client.index(index="test-index", doc_type="_doc", id="test-id", body={})
+        self.assert_url_called("PUT", "/test-index/_doc/test-id", count=2)
+
+    def test_termvectors(self):
+        self.client.termvectors(index="test-index", body={})
+        self.assert_url_called("POST", "/test-index/_termvectors")
+
+        self.client.termvectors(index="test-index", id="test-id", body={})
+        self.assert_url_called("POST", "/test-index/_termvectors/test-id")
+
+        self.client.termvectors(index="test-index", doc_type="test-type", body={})
+        self.assert_url_called("POST", "/test-index/test-type/_termvectors")
+
+        self.client.termvectors(
+            index="test-index", doc_type="test-type", id="test-id", body={}
+        )
+        self.assert_url_called("POST", "/test-index/test-type/test-id/_termvectors")
+
+    def test_mtermvectors(self):
+        self.client.mtermvectors(index="test-index", body={})
+        self.assert_url_called("POST", "/test-index/_mtermvectors")
+
+        self.client.mtermvectors(index="test-index", doc_type="test-type", body={})
+        self.assert_url_called("POST", "/test-index/test-type/_mtermvectors")
+
+    def test_update(self):
+        self.client.update(index="test-index", id="test-id", body={})
+        self.assert_url_called("POST", "/test-index/_update/test-id")
+
+        self.client.update(
+            index="test-index", doc_type="test-type", id="test-id", body={}
+        )
+        self.assert_url_called("POST", "/test-index/test-type/test-id/_update")
+
+    def test_cluster_state(self):
+        self.client.cluster.state()
+        self.assert_url_called("GET", "/_cluster/state")
+
+        self.client.cluster.state(index="test-index")
+        self.assert_url_called("GET", "/_cluster/state/_all/test-index")
+
+        self.client.cluster.state(index="test-index", metric="test-metric")
+        self.assert_url_called("GET", "/_cluster/state/test-metric/test-index")
+
+    def test_cluster_stats(self):
+        self.client.cluster.stats()
+        self.assert_url_called("GET", "/_cluster/stats")
+
+        self.client.cluster.stats(node_id="test-node")
+        self.assert_url_called("GET", "/_cluster/stats/nodes/test-node")
+
+    def test_indices_put_mapping(self):
+        self.client.indices.put_mapping(body={})
+        self.assert_url_called("PUT", "/_mapping")
+
+        self.client.indices.put_mapping(index="test-index", body={})
+        self.assert_url_called("PUT", "/test-index/_mapping")
+
+        self.client.indices.put_mapping(
+            index="test-index", doc_type="test-type", body={}
+        )
+        self.assert_url_called("PUT", "/test-index/test-type/_mapping")
+
+        self.client.indices.put_mapping(doc_type="test-type", body={})
+        self.assert_url_called("PUT", "/_all/test-type/_mapping")
+
+    def test_tasks_get(self):
+        with pytest.warns(DeprecationWarning):
+            self.client.tasks.get()

test_opensearchpy/test_client/test_utils.py

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from __future__ import unicode_literals
+
+from opensearchpy.client.utils import _bulk_body, _escape, _make_path, query_params
+from opensearchpy.compat import PY2
+
+from ..test_cases import SkipTest, TestCase
+
+
+class TestQueryParams(TestCase):
+    def setup_method(self, _):
+        self.calls = []
+
+    @query_params("simple_param")
+    def func_to_wrap(self, *args, **kwargs):
+        self.calls.append((args, kwargs))
+
+    def test_handles_params(self):
+        self.func_to_wrap(params={"simple_param_2": "2"}, simple_param="3")
+        self.assertEqual(
+            self.calls,
+            [
+                (
+                    (),
+                    {
+                        "params": {"simple_param": b"3", "simple_param_2": "2"},
+                        "headers": {},
+                    },
+                )
+            ],
+        )
+
+    def test_handles_headers(self):
+        self.func_to_wrap(headers={"X-Opaque-Id": "app-1"})
+        self.assertEqual(
+            self.calls, [((), {"params": {}, "headers": {"x-opaque-id": "app-1"}})]
+        )
+
+    def test_handles_opaque_id(self):
+        self.func_to_wrap(opaque_id="request-id")
+        self.assertEqual(
+            self.calls, [((), {"params": {}, "headers": {"x-opaque-id": "request-id"}})]
+        )
+
+    def test_handles_empty_none_and_normalization(self):
+        self.func_to_wrap(params=None)
+        self.assertEqual(self.calls[-1], ((), {"params": {}, "headers": {}}))
+
+        self.func_to_wrap(headers=None)
+        self.assertEqual(self.calls[-1], ((), {"params": {}, "headers": {}}))
+
+        self.func_to_wrap(headers=None, params=None)
+        self.assertEqual(self.calls[-1], ((), {"params": {}, "headers": {}}))
+
+        self.func_to_wrap(headers={}, params={})
+        self.assertEqual(self.calls[-1], ((), {"params": {}, "headers": {}}))
+
+        self.func_to_wrap(headers={"X": "y"})
+        self.assertEqual(self.calls[-1], ((), {"params": {}, "headers": {"x": "y"}}))
+
+    def test_per_call_authentication(self):
+        self.func_to_wrap(api_key=("name", "key"))
+        self.assertEqual(
+            self.calls[-1],
+            ((), {"headers": {"authorization": "ApiKey bmFtZTprZXk="}, "params": {}}),
+        )
+
+        self.func_to_wrap(http_auth=("user", "password"))
+        self.assertEqual(
+            self.calls[-1],
+            (
+                (),
+                {
+                    "headers": {"authorization": "Basic dXNlcjpwYXNzd29yZA=="},
+                    "params": {},
+                },
+            ),
+        )
+
+        self.func_to_wrap(http_auth="abcdef")
+        self.assertEqual(
+            self.calls[-1],
+            ((), {"headers": {"authorization": "Basic abcdef"}, "params": {}}),
+        )
+
+        # If one or the other is 'None' it's all good!
+        self.func_to_wrap(http_auth=None, api_key=None)
+        self.assertEqual(self.calls[-1], ((), {"headers": {}, "params": {}}))
+
+        self.func_to_wrap(http_auth="abcdef", api_key=None)
+        self.assertEqual(
+            self.calls[-1],
+            ((), {"headers": {"authorization": "Basic abcdef"}, "params": {}}),
+        )
+
+        # If both are given values an error is raised.
+        with self.assertRaises(ValueError) as e:
+            self.func_to_wrap(http_auth="key", api_key=("1", "2"))
+        self.assertEqual(
+            str(e.exception),
+            "Only one of 'http_auth' and 'api_key' may be passed at a time",
+        )
+
+
+class TestMakePath(TestCase):
+    def test_handles_unicode(self):
+        id = "中文"
+        self.assertEqual(
+            "/some-index/type/%E4%B8%AD%E6%96%87", _make_path("some-index", "type", id)
+        )
+
+    def test_handles_utf_encoded_string(self):
+        if not PY2:
+            raise SkipTest("Only relevant for py2")
+        id = "中文".encode("utf-8")
+        self.assertEqual(
+            "/some-index/type/%E4%B8%AD%E6%96%87", _make_path("some-index", "type", id)
+        )
+
+
+class TestEscape(TestCase):
+    def test_handles_ascii(self):
+        string = "abc123"
+        self.assertEqual(b"abc123", _escape(string))
+
+    def test_handles_unicode(self):
+        string = "中文"
+        self.assertEqual(b"\xe4\xb8\xad\xe6\x96\x87", _escape(string))
+
+    def test_handles_bytestring(self):
+        string = b"celery-task-meta-c4f1201f-eb7b-41d5-9318-a75a8cfbdaa0"
+        self.assertEqual(string, _escape(string))
+
+
+class TestBulkBody(TestCase):
+    def test_proper_bulk_body_as_string_is_not_modified(self):
+        string_body = '"{"index":{ "_index" : "test"}}\n{"field1": "value1"}"\n'
+        self.assertEqual(string_body, _bulk_body(None, string_body))
+
+    def test_proper_bulk_body_as_bytestring_is_not_modified(self):
+        bytestring_body = b'"{"index":{ "_index" : "test"}}\n{"field1": "value1"}"\n'
+        self.assertEqual(bytestring_body, _bulk_body(None, bytestring_body))
+
+    def test_bulk_body_as_string_adds_trailing_newline(self):
+        string_body = '"{"index":{ "_index" : "test"}}\n{"field1": "value1"}"'
+        self.assertEqual(
+            '"{"index":{ "_index" : "test"}}\n{"field1": "value1"}"\n',
+            _bulk_body(None, string_body),
+        )
+
+    def test_bulk_body_as_bytestring_adds_trailing_newline(self):
+        bytestring_body = b'"{"index":{ "_index" : "test"}}\n{"field1": "value1"}"'
+        self.assertEqual(
+            b'"{"index":{ "_index" : "test"}}\n{"field1": "value1"}"\n',
+            _bulk_body(None, bytestring_body),
+        )

test_opensearchpy/test_connection_pool.py

@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import time
+
+from opensearchpy.connection import Connection
+from opensearchpy.connection_pool import (
+    ConnectionPool,
+    DummyConnectionPool,
+    RoundRobinSelector,
+)
+from opensearchpy.exceptions import ImproperlyConfigured
+
+from .test_cases import TestCase
+
+
+class TestConnectionPool(TestCase):
+    def test_dummy_cp_raises_exception_on_more_connections(self):
+        self.assertRaises(ImproperlyConfigured, DummyConnectionPool, [])
+        self.assertRaises(
+            ImproperlyConfigured, DummyConnectionPool, [object(), object()]
+        )
+
+    def test_raises_exception_when_no_connections_defined(self):
+        self.assertRaises(ImproperlyConfigured, ConnectionPool, [])
+
+    def test_default_round_robin(self):
+        pool = ConnectionPool([(x, {}) for x in range(100)])
+
+        connections = set()
+        for _ in range(100):
+            connections.add(pool.get_connection())
+        self.assertEqual(connections, set(range(100)))
+
+    def test_disable_shuffling(self):
+        pool = ConnectionPool([(x, {}) for x in range(100)], randomize_hosts=False)
+
+        connections = []
+        for _ in range(100):
+            connections.append(pool.get_connection())
+        self.assertEqual(connections, list(range(100)))
+
+    def test_selectors_have_access_to_connection_opts(self):
+        class MySelector(RoundRobinSelector):
+            def select(self, connections):
+                return self.connection_opts[
+                    super(MySelector, self).select(connections)
+                ]["actual"]
+
+        pool = ConnectionPool(
+            [(x, {"actual": x * x}) for x in range(100)],
+            selector_class=MySelector,
+            randomize_hosts=False,
+        )
+
+        connections = []
+        for _ in range(100):
+            connections.append(pool.get_connection())
+        self.assertEqual(connections, [x * x for x in range(100)])
+
+    def test_dead_nodes_are_removed_from_active_connections(self):
+        pool = ConnectionPool([(x, {}) for x in range(100)])
+
+        now = time.time()
+        pool.mark_dead(42, now=now)
+        self.assertEqual(99, len(pool.connections))
+        self.assertEqual(1, pool.dead.qsize())
+        self.assertEqual((now + 60, 42), pool.dead.get())
+
+    def test_connection_is_skipped_when_dead(self):
+        pool = ConnectionPool([(x, {}) for x in range(2)])
+        pool.mark_dead(0)
+
+        self.assertEqual(
+            [1, 1, 1],
+            [pool.get_connection(), pool.get_connection(), pool.get_connection()],
+        )
+
+    def test_new_connection_is_not_marked_dead(self):
+        # Create 10 connections
+        pool = ConnectionPool([(Connection(), {}) for _ in range(10)])
+
+        # Pass in a new connection that is not in the pool to mark as dead
+        new_connection = Connection()
+        pool.mark_dead(new_connection)
+
+        # Nothing should be marked dead
+        self.assertEqual(0, len(pool.dead_count))
+
+    def test_connection_is_forcibly_resurrected_when_no_live_ones_are_availible(self):
+        pool = ConnectionPool([(x, {}) for x in range(2)])
+        pool.dead_count[0] = 1
+        pool.mark_dead(0)  # failed twice, longer timeout
+        pool.mark_dead(1)  # failed the first time, first to be resurrected
+
+        self.assertEqual([], pool.connections)
+        self.assertEqual(1, pool.get_connection())
+        self.assertEqual([1], pool.connections)
+
+    def test_connection_is_resurrected_after_its_timeout(self):
+        pool = ConnectionPool([(x, {}) for x in range(100)])
+
+        now = time.time()
+        pool.mark_dead(42, now=now - 61)
+        pool.get_connection()
+        self.assertEqual(42, pool.connections[-1])
+        self.assertEqual(100, len(pool.connections))
+
+    def test_force_resurrect_always_returns_a_connection(self):
+        pool = ConnectionPool([(0, {})])
+
+        pool.connections = []
+        self.assertEqual(0, pool.get_connection())
+        self.assertEqual([], pool.connections)
+        self.assertTrue(pool.dead.empty())
+
+    def test_already_failed_connection_has_longer_timeout(self):
+        pool = ConnectionPool([(x, {}) for x in range(100)])
+        now = time.time()
+        pool.dead_count[42] = 2
+        pool.mark_dead(42, now=now)
+
+        self.assertEqual(3, pool.dead_count[42])
+        self.assertEqual((now + 4 * 60, 42), pool.dead.get())
+
+    def test_timeout_for_failed_connections_is_limitted(self):
+        pool = ConnectionPool([(x, {}) for x in range(100)])
+        now = time.time()
+        pool.dead_count[42] = 245
+        pool.mark_dead(42, now=now)
+
+        self.assertEqual(246, pool.dead_count[42])
+        self.assertEqual((now + 32 * 60, 42), pool.dead.get())
+
+    def test_dead_count_is_wiped_clean_for_connection_if_marked_live(self):
+        pool = ConnectionPool([(x, {}) for x in range(100)])
+        now = time.time()
+        pool.dead_count[42] = 2
+        pool.mark_dead(42, now=now)
+
+        self.assertEqual(3, pool.dead_count[42])
+        pool.mark_live(42)
+        self.assertNotIn(42, pool.dead_count)

test_opensearchpy/test_exceptions.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from opensearchpy.exceptions import TransportError
+
+from .test_cases import TestCase
+
+
+class TestTransformError(TestCase):
+    def test_transform_error_parse_with_error_reason(self):
+        e = TransportError(
+            500,
+            "InternalServerError",
+            {"error": {"root_cause": [{"type": "error", "reason": "error reason"}]}},
+        )
+
+        self.assertEqual(
+            str(e), "TransportError(500, 'InternalServerError', 'error reason')"
+        )
+
+    def test_transform_error_parse_with_error_string(self):
+        e = TransportError(
+            500, "InternalServerError", {"error": "something error message"}
+        )
+
+        self.assertEqual(
+            str(e),
+            "TransportError(500, 'InternalServerError', 'something error message')",
+        )

test_opensearchpy/test_helpers.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import threading
+import time
+
+import mock
+import pytest
+
+from opensearchpy import OpenSearch, helpers
+from opensearchpy.serializer import JSONSerializer
+
+from .test_cases import TestCase
+
+lock_side_effect = threading.Lock()
+
+
+def mock_process_bulk_chunk(*args, **kwargs):
+    """
+    Threadsafe way of mocking process bulk chunk:
+    https://stackoverflow.com/questions/39332139/thread-safe-version-of-mock-call-count
+    """
+
+    with lock_side_effect:
+        mock_process_bulk_chunk.call_count += 1
+    time.sleep(0.1)
+    return []
+
+
+mock_process_bulk_chunk.call_count = 0
+
+
+class TestParallelBulk(TestCase):
+    @mock.patch(
+        "opensearchpy.helpers.actions._process_bulk_chunk",
+        side_effect=mock_process_bulk_chunk,
+    )
+    def test_all_chunks_sent(self, _process_bulk_chunk):
+        actions = ({"x": i} for i in range(100))
+        list(helpers.parallel_bulk(OpenSearch(), actions, chunk_size=2))
+
+        self.assertEqual(50, mock_process_bulk_chunk.call_count)
+
+    @pytest.mark.skip
+    @mock.patch(
+        "opensearchpy.helpers.actions._process_bulk_chunk",
+        # make sure we spend some time in the thread
+        side_effect=lambda *a: [
+            (True, time.sleep(0.001) or threading.current_thread().ident)
+        ],
+    )
+    def test_chunk_sent_from_different_threads(self, _process_bulk_chunk):
+        actions = ({"x": i} for i in range(100))
+        results = list(
+            helpers.parallel_bulk(OpenSearch(), actions, thread_count=10, chunk_size=2)
+        )
+        self.assertTrue(len(set([r[1] for r in results])) > 1)
+
+
+class TestChunkActions(TestCase):
+    def setup_method(self, _):
+        self.actions = [({"index": {}}, {"some": u"datá", "i": i}) for i in range(100)]
+
+    def test_expand_action(self):
+        self.assertEqual(helpers.expand_action({}), ({"index": {}}, {}))
+        self.assertEqual(
+            helpers.expand_action({"key": "val"}), ({"index": {}}, {"key": "val"})
+        )
+
+    def test_expand_action_actions(self):
+        self.assertEqual(
+            helpers.expand_action(
+                {"_op_type": "delete", "_id": "id", "_index": "index"}
+            ),
+            ({"delete": {"_id": "id", "_index": "index"}}, None),
+        )
+        self.assertEqual(
+            helpers.expand_action(
+                {"_op_type": "update", "_id": "id", "_index": "index", "key": "val"}
+            ),
+            ({"update": {"_id": "id", "_index": "index"}}, {"key": "val"}),
+        )
+        self.assertEqual(
+            helpers.expand_action(
+                {"_op_type": "create", "_id": "id", "_index": "index", "key": "val"}
+            ),
+            ({"create": {"_id": "id", "_index": "index"}}, {"key": "val"}),
+        )
+        self.assertEqual(
+            helpers.expand_action(
+                {
+                    "_op_type": "create",
+                    "_id": "id",
+                    "_index": "index",
+                    "_source": {"key": "val"},
+                }
+            ),
+            ({"create": {"_id": "id", "_index": "index"}}, {"key": "val"}),
+        )
+
+    def test_expand_action_options(self):
+        for option in (
+            "_id",
+            "_index",
+            "_percolate",
+            "_timestamp",
+            "_type",
+            "if_seq_no",
+            "if_primary_term",
+            "parent",
+            "pipeline",
+            "retry_on_conflict",
+            "routing",
+            "version",
+            "version_type",
+            ("_parent", "parent"),
+            ("_retry_on_conflict", "retry_on_conflict"),
+            ("_routing", "routing"),
+            ("_version", "version"),
+            ("_version_type", "version_type"),
+            ("_if_seq_no", "if_seq_no"),
+            ("_if_primary_term", "if_primary_term"),
+        ):
+            if isinstance(option, str):
+                action_option = option
+            else:
+                option, action_option = option
+            self.assertEqual(
+                helpers.expand_action({"key": "val", option: 0}),
+                ({"index": {action_option: 0}}, {"key": "val"}),
+            )
+
+    def test__source_metadata_or_source(self):
+        self.assertEqual(
+            helpers.expand_action({"_source": {"key": "val"}}),
+            ({"index": {}}, {"key": "val"}),
+        )
+
+        self.assertEqual(
+            helpers.expand_action(
+                {"_source": ["key"], "key": "val", "_op_type": "update"}
+            ),
+            ({"update": {"_source": ["key"]}}, {"key": "val"}),
+        )
+
+        self.assertEqual(
+            helpers.expand_action(
+                {"_source": True, "key": "val", "_op_type": "update"}
+            ),
+            ({"update": {"_source": True}}, {"key": "val"}),
+        )
+
+        # This case is only to ensure backwards compatibility with old functionality.
+        self.assertEqual(
+            helpers.expand_action(
+                {"_source": {"key2": "val2"}, "key": "val", "_op_type": "update"}
+            ),
+            ({"update": {}}, {"key2": "val2"}),
+        )
+
+    def test_chunks_are_chopped_by_byte_size(self):
+        self.assertEqual(
+            100,
+            len(
+                list(helpers._chunk_actions(self.actions, 100000, 1, JSONSerializer()))
+            ),
+        )
+
+    def test_chunks_are_chopped_by_chunk_size(self):
+        self.assertEqual(
+            10,
+            len(
+                list(
+                    helpers._chunk_actions(self.actions, 10, 99999999, JSONSerializer())
+                )
+            ),
+        )
+
+    def test_chunks_are_chopped_by_byte_size_properly(self):
+        max_byte_size = 170
+        chunks = list(
+            helpers._chunk_actions(
+                self.actions, 100000, max_byte_size, JSONSerializer()
+            )
+        )
+        self.assertEqual(25, len(chunks))
+        for chunk_data, chunk_actions in chunks:
+            chunk = u"".join(chunk_actions)
+            chunk = chunk if isinstance(chunk, str) else chunk.encode("utf-8")
+            self.assertLessEqual(len(chunk), max_byte_size)
+
+
+class TestExpandActions(TestCase):
+    def test_string_actions_are_marked_as_simple_inserts(self):
+        self.assertEqual(
+            ('{"index":{}}', "whatever"), helpers.expand_action("whatever")
+        )

test_opensearchpy/test_serializer.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import sys
+import uuid
+from datetime import datetime
+from decimal import Decimal
+
+try:
+    import numpy as np
+    import pandas as pd
+except ImportError:
+    np = pd = None
+
+from opensearchpy.exceptions import ImproperlyConfigured, SerializationError
+from opensearchpy.serializer import (
+    DEFAULT_SERIALIZERS,
+    Deserializer,
+    JSONSerializer,
+    TextSerializer,
+)
+
+from .test_cases import SkipTest, TestCase
+
+
+def requires_numpy_and_pandas():
+    if np is None or pd is None:
+        raise SkipTest("Test requires numpy or pandas to be available")
+
+
+class TestJSONSerializer(TestCase):
+    def test_datetime_serialization(self):
+        self.assertEqual(
+            '{"d":"2010-10-01T02:30:00"}',
+            JSONSerializer().dumps({"d": datetime(2010, 10, 1, 2, 30)}),
+        )
+
+    def test_decimal_serialization(self):
+        requires_numpy_and_pandas()
+
+        if sys.version_info[:2] == (2, 6):
+            raise SkipTest("Float rounding is broken in 2.6.")
+        self.assertEqual('{"d":3.8}', JSONSerializer().dumps({"d": Decimal("3.8")}))
+
+    def test_uuid_serialization(self):
+        self.assertEqual(
+            '{"d":"00000000-0000-0000-0000-000000000003"}',
+            JSONSerializer().dumps(
+                {"d": uuid.UUID("00000000-0000-0000-0000-000000000003")}
+            ),
+        )
+
+    def test_serializes_numpy_bool(self):
+        requires_numpy_and_pandas()
+
+        self.assertEqual('{"d":true}', JSONSerializer().dumps({"d": np.bool_(True)}))
+
+    def test_serializes_numpy_integers(self):
+        requires_numpy_and_pandas()
+
+        ser = JSONSerializer()
+        for np_type in (
+            np.int_,
+            np.int8,
+            np.int16,
+            np.int32,
+            np.int64,
+        ):
+            self.assertEqual(ser.dumps({"d": np_type(-1)}), '{"d":-1}')
+
+        for np_type in (
+            np.uint8,
+            np.uint16,
+            np.uint32,
+            np.uint64,
+        ):
+            self.assertEqual(ser.dumps({"d": np_type(1)}), '{"d":1}')
+
+    def test_serializes_numpy_floats(self):
+        requires_numpy_and_pandas()
+
+        ser = JSONSerializer()
+        for np_type in (
+            np.float_,
+            np.float32,
+            np.float64,
+        ):
+            self.assertRegexpMatches(
+                ser.dumps({"d": np_type(1.2)}), r'^\{"d":1\.2[\d]*}$'
+            )
+
+    def test_serializes_numpy_datetime(self):
+        requires_numpy_and_pandas()
+
+        self.assertEqual(
+            '{"d":"2010-10-01T02:30:00"}',
+            JSONSerializer().dumps({"d": np.datetime64("2010-10-01T02:30:00")}),
+        )
+
+    def test_serializes_numpy_ndarray(self):
+        requires_numpy_and_pandas()
+
+        self.assertEqual(
+            '{"d":[0,0,0,0,0]}',
+            JSONSerializer().dumps({"d": np.zeros((5,), dtype=np.uint8)}),
+        )
+        # This isn't useful for OpenSearch, just want to make sure it works.
+        self.assertEqual(
+            '{"d":[[0,0],[0,0]]}',
+            JSONSerializer().dumps({"d": np.zeros((2, 2), dtype=np.uint8)}),
+        )
+
+    def test_serializes_numpy_nan_to_nan(self):
+        requires_numpy_and_pandas()
+
+        self.assertEqual(
+            '{"d":NaN}',
+            JSONSerializer().dumps({"d": np.nan}),
+        )
+
+    def test_serializes_pandas_timestamp(self):
+        requires_numpy_and_pandas()
+
+        self.assertEqual(
+            '{"d":"2010-10-01T02:30:00"}',
+            JSONSerializer().dumps({"d": pd.Timestamp("2010-10-01T02:30:00")}),
+        )
+
+    def test_serializes_pandas_series(self):
+        requires_numpy_and_pandas()
+
+        self.assertEqual(
+            '{"d":["a","b","c","d"]}',
+            JSONSerializer().dumps({"d": pd.Series(["a", "b", "c", "d"])}),
+        )
+
+    def test_serializes_pandas_na(self):
+        requires_numpy_and_pandas()
+
+        if not hasattr(pd, "NA"):  # pandas.NA added in v1
+            raise SkipTest("pandas.NA required")
+        self.assertEqual(
+            '{"d":null}',
+            JSONSerializer().dumps({"d": pd.NA}),
+        )
+
+    def test_raises_serialization_error_pandas_nat(self):
+        requires_numpy_and_pandas()
+
+        if not hasattr(pd, "NaT"):
+            raise SkipTest("pandas.NaT required")
+        self.assertRaises(SerializationError, JSONSerializer().dumps, {"d": pd.NaT})
+
+    def test_serializes_pandas_category(self):
+        requires_numpy_and_pandas()
+
+        cat = pd.Categorical(["a", "c", "b", "a"], categories=["a", "b", "c"])
+        self.assertEqual(
+            '{"d":["a","c","b","a"]}',
+            JSONSerializer().dumps({"d": cat}),
+        )
+
+        cat = pd.Categorical([1, 2, 3], categories=[1, 2, 3])
+        self.assertEqual(
+            '{"d":[1,2,3]}',
+            JSONSerializer().dumps({"d": cat}),
+        )
+
+    def test_raises_serialization_error_on_dump_error(self):
+        self.assertRaises(SerializationError, JSONSerializer().dumps, object())
+
+    def test_raises_serialization_error_on_load_error(self):
+        self.assertRaises(SerializationError, JSONSerializer().loads, object())
+        self.assertRaises(SerializationError, JSONSerializer().loads, "")
+        self.assertRaises(SerializationError, JSONSerializer().loads, "{{")
+
+    def test_strings_are_left_untouched(self):
+        self.assertEqual("你好", JSONSerializer().dumps("你好"))
+
+
+class TestTextSerializer(TestCase):
+    def test_strings_are_left_untouched(self):
+        self.assertEqual("你好", TextSerializer().dumps("你好"))
+
+    def test_raises_serialization_error_on_dump_error(self):
+        self.assertRaises(SerializationError, TextSerializer().dumps, {})
+
+
+class TestDeserializer(TestCase):
+    def setup_method(self, _):
+        self.de = Deserializer(DEFAULT_SERIALIZERS)
+
+    def test_deserializes_json_by_default(self):
+        self.assertEqual({"some": "data"}, self.de.loads('{"some":"data"}'))
+
+    def test_deserializes_text_with_correct_ct(self):
+        self.assertEqual(
+            '{"some":"data"}', self.de.loads('{"some":"data"}', "text/plain")
+        )
+        self.assertEqual(
+            '{"some":"data"}',
+            self.de.loads('{"some":"data"}', "text/plain; charset=whatever"),
+        )
+
+    def test_raises_serialization_error_on_unknown_mimetype(self):
+        self.assertRaises(SerializationError, self.de.loads, "{}", "text/html")
+
+    def test_raises_improperly_configured_when_default_mimetype_cannot_be_deserialized(
+        self,
+    ):
+        self.assertRaises(ImproperlyConfigured, Deserializer, {})

test_opensearchpy/test_server/__init__.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from unittest import SkipTest
+
+from opensearchpy.helpers import test
+from opensearchpy.helpers.test import OpenSearchTestCase as BaseTestCase
+
+client = None
+
+
+def get_client(**kwargs):
+    global client
+    if client is False:
+        raise SkipTest("No client is available")
+    if client is not None and not kwargs:
+        return client
+
+    # try and locate manual override in the local environment
+    try:
+        from test_opensearchpy.local import get_client as local_get_client
+
+        new_client = local_get_client(**kwargs)
+    except ImportError:
+        # fallback to using vanilla client
+        try:
+            new_client = test.get_test_client(**kwargs)
+        except SkipTest:
+            client = False
+            raise
+
+    if not kwargs:
+        client = new_client
+
+    return new_client
+
+
+def setup_module():
+    get_client()
+
+
+class OpenSearchTestCase(BaseTestCase):
+    @staticmethod
+    def _get_client(**kwargs):
+        return get_client(**kwargs)

test_opensearchpy/test_server/conftest.py

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import os
+import time
+
+import pytest
+
+import opensearchpy
+from opensearchpy.helpers.test import CA_CERTS, OPENSEARCH_URL
+
+from ..utils import wipe_cluster
+
+# Information about the OpenSearch instance running, if any
+# Used for
+OPENSEARCH_VERSION = ""
+OPENSEARCH_BUILD_HASH = ""
+OPENSEARCH_REST_API_TESTS = []
+
+
+@pytest.fixture(scope="session")
+def sync_client_factory():
+    client = None
+    try:
+        # Configure the client with certificates and optionally
+        # an HTTP conn class depending on 'PYTHON_CONNECTION_CLASS' envvar
+        kw = {
+            "timeout": 3,
+            "ca_certs": CA_CERTS,
+            "headers": {"Authorization": "Basic ZWxhc3RpYzpjaGFuZ2VtZQ=="},
+        }
+        if "PYTHON_CONNECTION_CLASS" in os.environ:
+            from opensearchpy import connection
+
+            kw["connection_class"] = getattr(
+                connection, os.environ["PYTHON_CONNECTION_CLASS"]
+            )
+
+        # We do this little dance with the URL to force
+        # Requests to respect 'headers: None' within rest API spec tests.
+        client = opensearchpy.OpenSearch(
+            OPENSEARCH_URL.replace("elastic:changeme@", ""), **kw
+        )
+
+        # Wait for the cluster to report a status of 'yellow'
+        for _ in range(100):
+            try:
+                client.cluster.health(wait_for_status="yellow")
+                break
+            except ConnectionError:
+                time.sleep(0.1)
+        else:
+            pytest.skip("OpenSearch wasn't running at %r" % (OPENSEARCH_URL,))
+
+        wipe_cluster(client)
+        yield client
+    finally:
+        if client:
+            client.close()
+
+
+@pytest.fixture(scope="function")
+def sync_client(sync_client_factory):
+    try:
+        yield sync_client_factory
+    finally:
+        wipe_cluster(sync_client_factory)

test_opensearchpy/test_server/test_clients.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from __future__ import unicode_literals
+
+from . import OpenSearchTestCase
+
+
+class TestUnicode(OpenSearchTestCase):
+    def test_indices_analyze(self):
+        self.client.indices.analyze(body='{"text": "привет"}')
+
+
+class TestBulk(OpenSearchTestCase):
+    def test_bulk_works_with_string_body(self):
+        docs = '{ "index" : { "_index" : "bulk_test_index", "_id" : "1" } }\n{"answer": 42}'
+        response = self.client.bulk(body=docs)
+
+        self.assertFalse(response["errors"])
+        self.assertEqual(1, len(response["items"]))
+
+    def test_bulk_works_with_bytestring_body(self):
+        docs = b'{ "index" : { "_index" : "bulk_test_index", "_id" : "2" } }\n{"answer": 42}'
+        response = self.client.bulk(body=docs)
+
+        self.assertFalse(response["errors"])
+        self.assertEqual(1, len(response["items"]))

test_opensearchpy/test_server/test_helpers.py

@@ -0,0 +1,775 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from mock import patch
+
+from opensearchpy import TransportError, helpers
+from opensearchpy.helpers import ScanError
+
+from ..test_cases import SkipTest
+from . import OpenSearchTestCase
+
+
+class FailingBulkClient(object):
+    def __init__(
+        self, client, fail_at=(2,), fail_with=TransportError(599, "Error!", {})
+    ):
+        self.client = client
+        self._called = 0
+        self._fail_at = fail_at
+        self.transport = client.transport
+        self._fail_with = fail_with
+
+    def bulk(self, *args, **kwargs):
+        self._called += 1
+        if self._called in self._fail_at:
+            raise self._fail_with
+        return self.client.bulk(*args, **kwargs)
+
+
+class TestStreamingBulk(OpenSearchTestCase):
+    def test_actions_remain_unchanged(self):
+        actions = [{"_id": 1}, {"_id": 2}]
+        for ok, item in helpers.streaming_bulk(
+            self.client, actions, index="test-index"
+        ):
+            self.assertTrue(ok)
+        self.assertEqual([{"_id": 1}, {"_id": 2}], actions)
+
+    def test_all_documents_get_inserted(self):
+        docs = [{"answer": x, "_id": x} for x in range(100)]
+        for ok, item in helpers.streaming_bulk(
+            self.client, docs, index="test-index", refresh=True
+        ):
+            self.assertTrue(ok)
+
+        self.assertEqual(100, self.client.count(index="test-index")["count"])
+        self.assertEqual(
+            {"answer": 42}, self.client.get(index="test-index", id=42)["_source"]
+        )
+
+    def test_all_errors_from_chunk_are_raised_on_failure(self):
+        self.client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        self.client.cluster.health(wait_for_status="yellow")
+
+        try:
+            for ok, item in helpers.streaming_bulk(
+                self.client, [{"a": "b"}, {"a": "c"}], index="i", raise_on_error=True
+            ):
+                self.assertTrue(ok)
+        except helpers.BulkIndexError as e:
+            self.assertEqual(2, len(e.errors))
+        else:
+            assert False, "exception should have been raised"
+
+    def test_different_op_types(self):
+        if self.opensearch_version() < (0, 90, 1):
+            raise SkipTest("update supported since 0.90.1")
+        self.client.index(index="i", id=45, body={})
+        self.client.index(index="i", id=42, body={})
+        docs = [
+            {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
+            {"_op_type": "delete", "_index": "i", "_type": "_doc", "_id": 45},
+            {
+                "_op_type": "update",
+                "_index": "i",
+                "_type": "_doc",
+                "_id": 42,
+                "doc": {"answer": 42},
+            },
+        ]
+        for ok, item in helpers.streaming_bulk(self.client, docs):
+            self.assertTrue(ok)
+
+        self.assertFalse(self.client.exists(index="i", id=45))
+        self.assertEqual({"answer": 42}, self.client.get(index="i", id=42)["_source"])
+        self.assertEqual({"f": "v"}, self.client.get(index="i", id=47)["_source"])
+
+    def test_transport_error_can_becaught(self):
+        failing_client = FailingBulkClient(self.client)
+        docs = [
+            {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
+            {"_index": "i", "_type": "_doc", "_id": 45, "f": "v"},
+            {"_index": "i", "_type": "_doc", "_id": 42, "f": "v"},
+        ]
+
+        results = list(
+            helpers.streaming_bulk(
+                failing_client,
+                docs,
+                raise_on_exception=False,
+                raise_on_error=False,
+                chunk_size=1,
+            )
+        )
+        self.assertEqual(3, len(results))
+        self.assertEqual([True, False, True], [r[0] for r in results])
+
+        exc = results[1][1]["index"].pop("exception")
+        self.assertIsInstance(exc, TransportError)
+        self.assertEqual(599, exc.status_code)
+        self.assertEqual(
+            {
+                "index": {
+                    "_index": "i",
+                    "_type": "_doc",
+                    "_id": 45,
+                    "data": {"f": "v"},
+                    "error": "TransportError(599, 'Error!')",
+                    "status": 599,
+                }
+            },
+            results[1][1],
+        )
+
+    def test_rejected_documents_are_retried(self):
+        failing_client = FailingBulkClient(
+            self.client, fail_with=TransportError(429, "Rejected!", {})
+        )
+        docs = [
+            {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
+            {"_index": "i", "_type": "_doc", "_id": 45, "f": "v"},
+            {"_index": "i", "_type": "_doc", "_id": 42, "f": "v"},
+        ]
+        results = list(
+            helpers.streaming_bulk(
+                failing_client,
+                docs,
+                raise_on_exception=False,
+                raise_on_error=False,
+                chunk_size=1,
+                max_retries=1,
+                initial_backoff=0,
+            )
+        )
+        self.assertEqual(3, len(results))
+        self.assertEqual([True, True, True], [r[0] for r in results])
+        self.client.indices.refresh(index="i")
+        res = self.client.search(index="i")
+        self.assertEqual({"value": 3, "relation": "eq"}, res["hits"]["total"])
+        self.assertEqual(4, failing_client._called)
+
+    def test_rejected_documents_are_retried_at_most_max_retries_times(self):
+        failing_client = FailingBulkClient(
+            self.client, fail_at=(1, 2), fail_with=TransportError(429, "Rejected!", {})
+        )
+
+        docs = [
+            {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
+            {"_index": "i", "_type": "_doc", "_id": 45, "f": "v"},
+            {"_index": "i", "_type": "_doc", "_id": 42, "f": "v"},
+        ]
+        results = list(
+            helpers.streaming_bulk(
+                failing_client,
+                docs,
+                raise_on_exception=False,
+                raise_on_error=False,
+                chunk_size=1,
+                max_retries=1,
+                initial_backoff=0,
+            )
+        )
+        self.assertEqual(3, len(results))
+        self.assertEqual([False, True, True], [r[0] for r in results])
+        self.client.indices.refresh(index="i")
+        res = self.client.search(index="i")
+        self.assertEqual({"value": 2, "relation": "eq"}, res["hits"]["total"])
+        self.assertEqual(4, failing_client._called)
+
+    def test_transport_error_is_raised_with_max_retries(self):
+        failing_client = FailingBulkClient(
+            self.client,
+            fail_at=(1, 2, 3, 4),
+            fail_with=TransportError(429, "Rejected!", {}),
+        )
+
+        def streaming_bulk():
+            results = list(
+                helpers.streaming_bulk(
+                    failing_client,
+                    [{"a": 42}, {"a": 39}],
+                    raise_on_exception=True,
+                    max_retries=3,
+                    initial_backoff=0,
+                )
+            )
+            return results
+
+        self.assertRaises(TransportError, streaming_bulk)
+        self.assertEqual(4, failing_client._called)
+
+
+class TestBulk(OpenSearchTestCase):
+    def test_bulk_works_with_single_item(self):
+        docs = [{"answer": 42, "_id": 1}]
+        success, failed = helpers.bulk(
+            self.client, docs, index="test-index", refresh=True
+        )
+
+        self.assertEqual(1, success)
+        self.assertFalse(failed)
+        self.assertEqual(1, self.client.count(index="test-index")["count"])
+        self.assertEqual(
+            {"answer": 42}, self.client.get(index="test-index", id=1)["_source"]
+        )
+
+    def test_all_documents_get_inserted(self):
+        docs = [{"answer": x, "_id": x} for x in range(100)]
+        success, failed = helpers.bulk(
+            self.client, docs, index="test-index", refresh=True
+        )
+
+        self.assertEqual(100, success)
+        self.assertFalse(failed)
+        self.assertEqual(100, self.client.count(index="test-index")["count"])
+        self.assertEqual(
+            {"answer": 42}, self.client.get(index="test-index", id=42)["_source"]
+        )
+
+    def test_stats_only_reports_numbers(self):
+        docs = [{"answer": x} for x in range(100)]
+        success, failed = helpers.bulk(
+            self.client, docs, index="test-index", refresh=True, stats_only=True
+        )
+
+        self.assertEqual(100, success)
+        self.assertEqual(0, failed)
+        self.assertEqual(100, self.client.count(index="test-index")["count"])
+
+    def test_errors_are_reported_correctly(self):
+        self.client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        self.client.cluster.health(wait_for_status="yellow")
+
+        success, failed = helpers.bulk(
+            self.client,
+            [{"a": 42}, {"a": "c", "_id": 42}],
+            index="i",
+            raise_on_error=False,
+        )
+        self.assertEqual(1, success)
+        self.assertEqual(1, len(failed))
+        error = failed[0]
+        self.assertEqual("42", error["index"]["_id"])
+        self.assertEqual("_doc", error["index"]["_type"])
+        self.assertEqual("i", error["index"]["_index"])
+        print(error["index"]["error"])
+        self.assertTrue(
+            "MapperParsingException" in repr(error["index"]["error"])
+            or "mapper_parsing_exception" in repr(error["index"]["error"])
+        )
+
+    def test_error_is_raised(self):
+        self.client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        self.client.cluster.health(wait_for_status="yellow")
+
+        self.assertRaises(
+            helpers.BulkIndexError,
+            helpers.bulk,
+            self.client,
+            [{"a": 42}, {"a": "c"}],
+            index="i",
+        )
+
+    def test_ignore_error_if_raised(self):
+        # ignore the status code 400 in tuple
+        helpers.bulk(
+            self.client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=(400,)
+        )
+
+        # ignore the status code 400 in list
+        helpers.bulk(
+            self.client,
+            [{"a": 42}, {"a": "c"}],
+            index="i",
+            ignore_status=[
+                400,
+            ],
+        )
+
+        # ignore the status code 400
+        helpers.bulk(self.client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=400)
+
+        # ignore only the status code in the `ignore_status` argument
+        self.assertRaises(
+            helpers.BulkIndexError,
+            helpers.bulk,
+            self.client,
+            [{"a": 42}, {"a": "c"}],
+            index="i",
+            ignore_status=(444,),
+        )
+
+        # ignore transport error exception
+        failing_client = FailingBulkClient(self.client)
+        helpers.bulk(failing_client, [{"a": 42}], index="i", ignore_status=(599,))
+
+    def test_errors_are_collected_properly(self):
+        self.client.indices.create(
+            "i",
+            {
+                "mappings": {"properties": {"a": {"type": "integer"}}},
+                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            },
+        )
+        self.client.cluster.health(wait_for_status="yellow")
+
+        success, failed = helpers.bulk(
+            self.client,
+            [{"a": 42}, {"a": "c"}],
+            index="i",
+            stats_only=True,
+            raise_on_error=False,
+        )
+        self.assertEqual(1, success)
+        self.assertEqual(1, failed)
+
+
+class TestScan(OpenSearchTestCase):
+    mock_scroll_responses = [
+        {
+            "_scroll_id": "dummy_id",
+            "_shards": {"successful": 4, "total": 5, "skipped": 0},
+            "hits": {"hits": [{"scroll_data": 42}]},
+        },
+        {
+            "_scroll_id": "dummy_id",
+            "_shards": {"successful": 4, "total": 5, "skipped": 0},
+            "hits": {"hits": []},
+        },
+    ]
+
+    def teardown_method(self, m):
+        self.client.transport.perform_request("DELETE", "/_search/scroll/_all")
+        super(TestScan, self).teardown_method(m)
+
+    def test_order_can_be_preserved(self):
+        bulk = []
+        for x in range(100):
+            bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}})
+            bulk.append({"answer": x, "correct": x == 42})
+        self.client.bulk(bulk, refresh=True)
+
+        docs = list(
+            helpers.scan(
+                self.client,
+                index="test_index",
+                query={"sort": "answer"},
+                preserve_order=True,
+            )
+        )
+
+        self.assertEqual(100, len(docs))
+        self.assertEqual(list(map(str, range(100))), list(d["_id"] for d in docs))
+        self.assertEqual(list(range(100)), list(d["_source"]["answer"] for d in docs))
+
+    def test_all_documents_are_read(self):
+        bulk = []
+        for x in range(100):
+            bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}})
+            bulk.append({"answer": x, "correct": x == 42})
+        self.client.bulk(bulk, refresh=True)
+
+        docs = list(helpers.scan(self.client, index="test_index", size=2))
+
+        self.assertEqual(100, len(docs))
+        self.assertEqual(set(map(str, range(100))), set(d["_id"] for d in docs))
+        self.assertEqual(set(range(100)), set(d["_source"]["answer"] for d in docs))
+
+    def test_scroll_error(self):
+        bulk = []
+        for x in range(4):
+            bulk.append({"index": {"_index": "test_index", "_type": "_doc"}})
+            bulk.append({"value": x})
+        self.client.bulk(bulk, refresh=True)
+
+        with patch.object(self.client, "scroll") as scroll_mock:
+            scroll_mock.side_effect = self.mock_scroll_responses
+            data = list(
+                helpers.scan(
+                    self.client,
+                    index="test_index",
+                    size=2,
+                    raise_on_error=False,
+                    clear_scroll=False,
+                )
+            )
+            self.assertEqual(len(data), 3)
+            self.assertEqual(data[-1], {"scroll_data": 42})
+
+            scroll_mock.side_effect = self.mock_scroll_responses
+            with self.assertRaises(ScanError):
+                data = list(
+                    helpers.scan(
+                        self.client,
+                        index="test_index",
+                        size=2,
+                        raise_on_error=True,
+                        clear_scroll=False,
+                    )
+                )
+            self.assertEqual(len(data), 3)
+            self.assertEqual(data[-1], {"scroll_data": 42})
+
+    def test_initial_search_error(self):
+        with patch.object(self, "client") as client_mock:
+            client_mock.search.return_value = {
+                "_scroll_id": "dummy_id",
+                "_shards": {"successful": 4, "total": 5, "skipped": 0},
+                "hits": {"hits": [{"search_data": 1}]},
+            }
+            client_mock.scroll.side_effect = self.mock_scroll_responses
+
+            data = list(
+                helpers.scan(
+                    self.client, index="test_index", size=2, raise_on_error=False
+                )
+            )
+            self.assertEqual(data, [{"search_data": 1}, {"scroll_data": 42}])
+
+            client_mock.scroll.side_effect = self.mock_scroll_responses
+            with self.assertRaises(ScanError):
+                data = list(
+                    helpers.scan(
+                        self.client, index="test_index", size=2, raise_on_error=True
+                    )
+                )
+                self.assertEqual(data, [{"search_data": 1}])
+                client_mock.scroll.assert_not_called()
+
+    def test_no_scroll_id_fast_route(self):
+        with patch.object(self, "client") as client_mock:
+            client_mock.search.return_value = {"no": "_scroll_id"}
+            data = list(helpers.scan(self.client, index="test_index"))
+
+            self.assertEqual(data, [])
+            client_mock.scroll.assert_not_called()
+            client_mock.clear_scroll.assert_not_called()
+
+    def test_scan_auth_kwargs_forwarded(self):
+        for key, val in {
+            "api_key": ("name", "value"),
+            "http_auth": ("username", "password"),
+            "headers": {"custom": "header"},
+        }.items():
+            with patch.object(self, "client") as client_mock:
+                client_mock.search.return_value = {
+                    "_scroll_id": "scroll_id",
+                    "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                    "hits": {"hits": [{"search_data": 1}]},
+                }
+                client_mock.scroll.return_value = {
+                    "_scroll_id": "scroll_id",
+                    "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                    "hits": {"hits": []},
+                }
+                client_mock.clear_scroll.return_value = {}
+
+                data = list(helpers.scan(self.client, index="test_index", **{key: val}))
+
+                self.assertEqual(data, [{"search_data": 1}])
+
+                # Assert that 'search', 'scroll' and 'clear_scroll' all
+                # received the extra kwarg related to authentication.
+                for api_mock in (
+                    client_mock.search,
+                    client_mock.scroll,
+                    client_mock.clear_scroll,
+                ):
+                    self.assertEqual(api_mock.call_args[1][key], val)
+
+    def test_scan_auth_kwargs_favor_scroll_kwargs_option(self):
+        with patch.object(self, "client") as client_mock:
+            client_mock.search.return_value = {
+                "_scroll_id": "scroll_id",
+                "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                "hits": {"hits": [{"search_data": 1}]},
+            }
+            client_mock.scroll.return_value = {
+                "_scroll_id": "scroll_id",
+                "_shards": {"successful": 5, "total": 5, "skipped": 0},
+                "hits": {"hits": []},
+            }
+            client_mock.clear_scroll.return_value = {}
+
+            data = list(
+                helpers.scan(
+                    self.client,
+                    index="test_index",
+                    scroll_kwargs={"headers": {"scroll": "kwargs"}, "sort": "asc"},
+                    headers={"not scroll": "kwargs"},
+                )
+            )
+
+            self.assertEqual(data, [{"search_data": 1}])
+
+            # Assert that we see 'scroll_kwargs' options used instead of 'kwargs'
+            self.assertEqual(
+                client_mock.scroll.call_args[1]["headers"], {"scroll": "kwargs"}
+            )
+            self.assertEqual(client_mock.scroll.call_args[1]["sort"], "asc")
+
+    @patch("opensearchpy.helpers.actions.logger")
+    def test_logger(self, logger_mock):
+        bulk = []
+        for x in range(4):
+            bulk.append({"index": {"_index": "test_index", "_type": "_doc"}})
+            bulk.append({"value": x})
+        self.client.bulk(bulk, refresh=True)
+
+        with patch.object(self.client, "scroll") as scroll_mock:
+            scroll_mock.side_effect = self.mock_scroll_responses
+            list(
+                helpers.scan(
+                    self.client,
+                    index="test_index",
+                    size=2,
+                    raise_on_error=False,
+                    clear_scroll=False,
+                )
+            )
+            logger_mock.warning.assert_called()
+
+            scroll_mock.side_effect = self.mock_scroll_responses
+            try:
+                list(
+                    helpers.scan(
+                        self.client,
+                        index="test_index",
+                        size=2,
+                        raise_on_error=True,
+                        clear_scroll=False,
+                    )
+                )
+            except ScanError:
+                pass
+            logger_mock.warning.assert_called()
+
+    def test_clear_scroll(self):
+        bulk = []
+        for x in range(4):
+            bulk.append({"index": {"_index": "test_index", "_type": "_doc"}})
+            bulk.append({"value": x})
+        self.client.bulk(bulk, refresh=True)
+
+        with patch.object(
+            self.client, "clear_scroll", wraps=self.client.clear_scroll
+        ) as spy:
+            list(helpers.scan(self.client, index="test_index", size=2))
+            spy.assert_called_once()
+
+            spy.reset_mock()
+            list(
+                helpers.scan(self.client, index="test_index", size=2, clear_scroll=True)
+            )
+            spy.assert_called_once()
+
+            spy.reset_mock()
+            list(
+                helpers.scan(
+                    self.client, index="test_index", size=2, clear_scroll=False
+                )
+            )
+            spy.assert_not_called()
+
+    def test_shards_no_skipped_field(self):
+        with patch.object(self, "client") as client_mock:
+            client_mock.search.return_value = {
+                "_scroll_id": "dummy_id",
+                "_shards": {"successful": 5, "total": 5},
+                "hits": {"hits": [{"search_data": 1}]},
+            }
+            client_mock.scroll.side_effect = [
+                {
+                    "_scroll_id": "dummy_id",
+                    "_shards": {"successful": 5, "total": 5},
+                    "hits": {"hits": [{"scroll_data": 42}]},
+                },
+                {
+                    "_scroll_id": "dummy_id",
+                    "_shards": {"successful": 5, "total": 5},
+                    "hits": {"hits": []},
+                },
+            ]
+
+            data = list(
+                helpers.scan(
+                    self.client, index="test_index", size=2, raise_on_error=True
+                )
+            )
+            self.assertEqual(data, [{"search_data": 1}, {"scroll_data": 42}])
+
+
+class TestReindex(OpenSearchTestCase):
+    def setup_method(self, _):
+        bulk = []
+        for x in range(100):
+            bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}})
+            bulk.append(
+                {
+                    "answer": x,
+                    "correct": x == 42,
+                    "type": "answers" if x % 2 == 0 else "questions",
+                }
+            )
+        self.client.bulk(bulk, refresh=True)
+
+    def test_reindex_passes_kwargs_to_scan_and_bulk(self):
+        helpers.reindex(
+            self.client,
+            "test_index",
+            "prod_index",
+            scan_kwargs={"q": "type:answers"},
+            bulk_kwargs={"refresh": True},
+        )
+
+        self.assertTrue(self.client.indices.exists("prod_index"))
+        self.assertEqual(
+            50, self.client.count(index="prod_index", q="type:answers")["count"]
+        )
+
+        self.assertEqual(
+            {"answer": 42, "correct": True, "type": "answers"},
+            self.client.get(index="prod_index", id=42)["_source"],
+        )
+
+    def test_reindex_accepts_a_query(self):
+        helpers.reindex(
+            self.client,
+            "test_index",
+            "prod_index",
+            query={"query": {"bool": {"filter": {"term": {"type": "answers"}}}}},
+        )
+        self.client.indices.refresh()
+
+        self.assertTrue(self.client.indices.exists("prod_index"))
+        self.assertEqual(
+            50, self.client.count(index="prod_index", q="type:answers")["count"]
+        )
+
+        self.assertEqual(
+            {"answer": 42, "correct": True, "type": "answers"},
+            self.client.get(index="prod_index", id=42)["_source"],
+        )
+
+    def test_all_documents_get_moved(self):
+        helpers.reindex(self.client, "test_index", "prod_index")
+        self.client.indices.refresh()
+
+        self.assertTrue(self.client.indices.exists("prod_index"))
+        self.assertEqual(
+            50, self.client.count(index="prod_index", q="type:questions")["count"]
+        )
+        self.assertEqual(
+            50, self.client.count(index="prod_index", q="type:answers")["count"]
+        )
+
+        self.assertEqual(
+            {"answer": 42, "correct": True, "type": "answers"},
+            self.client.get(index="prod_index", id=42)["_source"],
+        )
+
+
+class TestParentChildReindex(OpenSearchTestCase):
+    def setup_method(self, _):
+        body = {
+            "settings": {"number_of_shards": 1, "number_of_replicas": 0},
+            "mappings": {
+                "properties": {
+                    "question_answer": {
+                        "type": "join",
+                        "relations": {"question": "answer"},
+                    }
+                }
+            },
+        }
+        self.client.indices.create(index="test-index", body=body)
+        self.client.indices.create(index="real-index", body=body)
+
+        self.client.index(
+            index="test-index", id=42, body={"question_answer": "question"}
+        )
+        self.client.index(
+            index="test-index",
+            id=47,
+            routing=42,
+            body={"some": "data", "question_answer": {"name": "answer", "parent": 42}},
+        )
+        self.client.indices.refresh(index="test-index")
+
+    def test_children_are_reindexed_correctly(self):
+        helpers.reindex(self.client, "test-index", "real-index")
+
+        q = self.client.get(index="real-index", id=42)
+        self.assertEqual(
+            {
+                "_id": "42",
+                "_index": "real-index",
+                "_primary_term": 1,
+                "_seq_no": 0,
+                "_source": {"question_answer": "question"},
+                "_type": "_doc",
+                "_version": 1,
+                "found": True,
+            },
+            q,
+        )
+        q = self.client.get(index="test-index", id=47, routing=42)
+        self.assertEqual(
+            {
+                "_routing": "42",
+                "_id": "47",
+                "_index": "test-index",
+                "_primary_term": 1,
+                "_seq_no": 1,
+                "_source": {
+                    "some": "data",
+                    "question_answer": {"name": "answer", "parent": 42},
+                },
+                "_type": "_doc",
+                "_version": 1,
+                "found": True,
+            },
+            q,
+        )

test_opensearchpy/test_server/test_rest_api_spec.py

@@ -0,0 +1,566 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+"""
+Dynamically generated set of TestCases based on set of yaml files describing
+some integration tests. These files are shared among all official OpenSearch
+clients.
+"""
+import io
+import json
+import os
+import re
+import sys
+import warnings
+import zipfile
+
+import pytest
+import urllib3
+import yaml
+
+from opensearchpy import OpenSearchWarning, TransportError
+from opensearchpy.client.utils import _base64_auth_header
+from opensearchpy.compat import string_types
+from opensearchpy.helpers.test import _get_version
+
+from . import get_client
+
+# some params had to be changed in python, keep track of them so we can rename
+# those in the tests accordingly
+PARAMS_RENAMES = {"type": "doc_type", "from": "from_"}
+
+# mapping from catch values to http status codes
+CATCH_CODES = {"missing": 404, "conflict": 409, "unauthorized": 401}
+
+# test features we have implemented
+IMPLEMENTED_FEATURES = {
+    "gtelte",
+    "stash_in_path",
+    "headers",
+    "catch_unauthorized",
+    "default_shards",
+    "warnings",
+    "allowed_warnings",
+    "contains",
+    "arbitrary_key",
+    "transform_and_set",
+}
+
+# broken YAML tests on some releases
+SKIP_TESTS = {
+    # Warning about date_histogram.interval deprecation is raised randomly
+    "search/aggregation/250_moving_fn[1]",
+    # body: null
+    "indices/simulate_index_template/10_basic[2]",
+    # No ML node with sufficient capacity / random ML failing
+    "ml/start_stop_datafeed",
+    "ml/post_data",
+    "ml/jobs_crud",
+    "ml/datafeeds_crud",
+    "ml/set_upgrade_mode",
+    "ml/reset_job[2]",
+    "ml/jobs_get_stats",
+    "ml/get_datafeed_stats",
+    "ml/get_trained_model_stats",
+    "ml/delete_job_force",
+    "ml/jobs_get_result_overall_buckets",
+    "ml/bucket_correlation_agg[0]",
+    "ml/job_groups",
+    "transform/transforms_stats_continuous[0]",
+    # Fails bad request instead of 404?
+    "ml/inference_crud",
+    # Our TLS certs are custom
+    "ssl/10_basic[0]",
+    # Our user is custom
+    "users/10_basic[3]",
+    # Shards/snapshots aren't right?
+    "searchable_snapshots/10_usage[1]",
+    # flaky data streams?
+    "data_stream/10_basic[1]",
+    "data_stream/80_resolve_index_data_streams[1]",
+    # bad formatting?
+    "cat/allocation/10_basic",
+    # service account number not right?
+    "service_accounts/10_basic[1]",
+    # doesn't use 'contains' properly?
+    "privileges/40_get_user_privs[0]",
+    "privileges/40_get_user_privs[1]",
+    # bad use of 'is_false'?
+    "indices/get_alias/10_basic[22]",
+    # unique usage of 'set'
+    "indices/stats/50_disk_usage[0]",
+    "indices/stats/60_field_usage[0]",
+}
+
+
+OPENSEARCH_VERSION = None
+RUN_ASYNC_REST_API_TESTS = (
+    sys.version_info >= (3, 6)
+    and os.environ.get("PYTHON_CONNECTION_CLASS") == "RequestsHttpConnection"
+)
+
+FALSEY_VALUES = ("", None, False, 0, 0.0)
+
+
+class YamlRunner:
+    def __init__(self, client):
+        self.client = client
+        self.last_response = None
+
+        self._run_code = None
+        self._setup_code = None
+        self._teardown_code = None
+        self._state = {}
+
+    def use_spec(self, test_spec):
+        self._setup_code = test_spec.pop("setup", None)
+        self._run_code = test_spec.pop("run", None)
+        self._teardown_code = test_spec.pop("teardown", None)
+
+    def setup(self):
+        # Pull skips from individual tests to not do unnecessary setup.
+        skip_code = []
+        for action in self._run_code:
+            assert len(action) == 1
+            action_type, _ = list(action.items())[0]
+            if action_type == "skip":
+                skip_code.append(action)
+            else:
+                break
+
+        if self._setup_code or skip_code:
+            self.section("setup")
+        if skip_code:
+            self.run_code(skip_code)
+        if self._setup_code:
+            self.run_code(self._setup_code)
+
+    def teardown(self):
+        if self._teardown_code:
+            self.section("teardown")
+            self.run_code(self._teardown_code)
+
+    def opensearch_version(self):
+        global OPENSEARCH_VERSION
+        if OPENSEARCH_VERSION is None:
+            version_string = (self.client.info())["version"]["number"]
+            if "." not in version_string:
+                return ()
+            version = version_string.strip().split(".")
+            OPENSEARCH_VERSION = tuple(int(v) if v.isdigit() else 999 for v in version)
+        return OPENSEARCH_VERSION
+
+    def section(self, name):
+        print(("=" * 10) + " " + name + " " + ("=" * 10))
+
+    def run(self):
+        try:
+            self.setup()
+            self.section("test")
+            self.run_code(self._run_code)
+        finally:
+            try:
+                self.teardown()
+            except Exception:
+                pass
+
+    def run_code(self, test):
+        """Execute an instruction based on it's type."""
+        for action in test:
+            assert len(action) == 1
+            action_type, action = list(action.items())[0]
+            print(action_type, action)
+
+            if hasattr(self, "run_" + action_type):
+                getattr(self, "run_" + action_type)(action)
+            else:
+                raise RuntimeError("Invalid action type %r" % (action_type,))
+
+    def run_do(self, action):
+        api = self.client
+        headers = action.pop("headers", None)
+        catch = action.pop("catch", None)
+        warn = action.pop("warnings", ())
+        allowed_warnings = action.pop("allowed_warnings", ())
+        assert len(action) == 1
+
+        # Remove the x_pack_rest_user authentication
+        # if it's given via headers. We're already authenticated
+        # via the 'elastic' user.
+        if (
+            headers
+            and headers.get("Authorization", None)
+            == "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA=="
+        ):
+            headers.pop("Authorization")
+
+        method, args = list(action.items())[0]
+        args["headers"] = headers
+
+        # locate api endpoint
+        for m in method.split("."):
+            assert hasattr(api, m)
+            api = getattr(api, m)
+
+        # some parameters had to be renamed to not clash with python builtins,
+        # compensate
+        for k in PARAMS_RENAMES:
+            if k in args:
+                args[PARAMS_RENAMES[k]] = args.pop(k)
+
+        # resolve vars
+        for k in args:
+            args[k] = self._resolve(args[k])
+
+        warnings.simplefilter("always", category=OpenSearchWarning)
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            try:
+                self.last_response = api(**args)
+            except Exception as e:
+                if not catch:
+                    raise
+                self.run_catch(catch, e)
+            else:
+                if catch:
+                    raise AssertionError(
+                        "Failed to catch %r in %r." % (catch, self.last_response)
+                    )
+
+        # Filter out warnings raised by other components.
+        caught_warnings = [
+            str(w.message)
+            for w in caught_warnings
+            if w.category == OpenSearchWarning
+            and str(w.message) not in allowed_warnings
+        ]
+
+        # This warning can show up in many places but isn't accounted for
+        # in tests, so we remove it to make sure things pass.
+        include_type_name_warning = (
+            "[types removal] Using include_type_name in create index requests is deprecated. "
+            "The parameter will be removed in the next major version."
+        )
+        if (
+            include_type_name_warning in caught_warnings
+            and include_type_name_warning not in warn
+        ):
+            caught_warnings.remove(include_type_name_warning)
+
+        # Sorting removes the issue with order raised. We only care about
+        # if all warnings are raised in the single API call.
+        if warn and sorted(warn) != sorted(caught_warnings):
+            raise AssertionError(
+                "Expected warnings not equal to actual warnings: expected=%r actual=%r"
+                % (warn, caught_warnings)
+            )
+
+    def run_catch(self, catch, exception):
+        if catch == "param":
+            assert isinstance(exception, TypeError)
+            return
+
+        assert isinstance(exception, TransportError)
+        if catch in CATCH_CODES:
+            assert CATCH_CODES[catch] == exception.status_code
+        elif catch[0] == "/" and catch[-1] == "/":
+            assert (
+                re.search(catch[1:-1], exception.error + " " + repr(exception.info)),
+                "%s not in %r" % (catch, exception.info),
+            ) is not None
+        self.last_response = exception.info
+
+    def run_skip(self, skip):
+        global IMPLEMENTED_FEATURES
+
+        if "features" in skip:
+            features = skip["features"]
+            if not isinstance(features, (tuple, list)):
+                features = [features]
+            for feature in features:
+                if feature in IMPLEMENTED_FEATURES:
+                    continue
+                pytest.skip("feature '%s' is not supported" % feature)
+
+        if "version" in skip:
+            version, reason = skip["version"], skip["reason"]
+            if version == "all":
+                pytest.skip(reason)
+            min_version, max_version = version.split("-")
+            min_version = _get_version(min_version) or (0,)
+            max_version = _get_version(max_version) or (999,)
+            if min_version <= (self.opensearch_version()) <= max_version:
+                pytest.skip(reason)
+
+    def run_gt(self, action):
+        for key, value in action.items():
+            value = self._resolve(value)
+            assert self._lookup(key) > value
+
+    def run_gte(self, action):
+        for key, value in action.items():
+            value = self._resolve(value)
+            assert self._lookup(key) >= value
+
+    def run_lt(self, action):
+        for key, value in action.items():
+            value = self._resolve(value)
+            assert self._lookup(key) < value
+
+    def run_lte(self, action):
+        for key, value in action.items():
+            value = self._resolve(value)
+            assert self._lookup(key) <= value
+
+    def run_set(self, action):
+        for key, value in action.items():
+            value = self._resolve(value)
+            self._state[value] = self._lookup(key)
+
+    def run_is_false(self, action):
+        try:
+            value = self._lookup(action)
+        except AssertionError:
+            pass
+        else:
+            assert value in FALSEY_VALUES
+
+    def run_is_true(self, action):
+        value = self._lookup(action)
+        assert value not in FALSEY_VALUES
+
+    def run_length(self, action):
+        for path, expected in action.items():
+            value = self._lookup(path)
+            expected = self._resolve(expected)
+            assert expected == len(value)
+
+    def run_match(self, action):
+        for path, expected in action.items():
+            value = self._lookup(path)
+            expected = self._resolve(expected)
+
+            if (
+                isinstance(expected, string_types)
+                and expected.startswith("/")
+                and expected.endswith("/")
+            ):
+                expected = re.compile(expected[1:-1], re.VERBOSE | re.MULTILINE)
+                assert expected.search(value), "%r does not match %r" % (
+                    value,
+                    expected,
+                )
+            else:
+                self._assert_match_equals(value, expected)
+
+    def run_contains(self, action):
+        for path, expected in action.items():
+            value = self._lookup(path)  # list[dict[str,str]] is returned
+            expected = self._resolve(expected)  # dict[str, str]
+
+            if expected not in value:
+                raise AssertionError("%s is not contained by %s" % (expected, value))
+
+    def run_transform_and_set(self, action):
+        for key, value in action.items():
+            # Convert #base64EncodeCredentials(id,api_key) to ["id", "api_key"]
+            if "#base64EncodeCredentials" in value:
+                value = value.replace("#base64EncodeCredentials", "")
+                value = value.replace("(", "").replace(")", "").split(",")
+                self._state[key] = _base64_auth_header(
+                    (self._lookup(value[0]), self._lookup(value[1]))
+                )
+
+    def _resolve(self, value):
+        # resolve variables
+        if isinstance(value, string_types) and "$" in value:
+            for k, v in self._state.items():
+                for key_replace in ("${" + k + "}", "$" + k):
+                    if value == key_replace:
+                        value = v
+                        break
+                    # We only do the in-string replacement if using ${...}
+                    elif (
+                        key_replace.startswith("${")
+                        and isinstance(value, string_types)
+                        and key_replace in value
+                    ):
+                        value = value.replace(key_replace, v)
+                        break
+
+        if isinstance(value, string_types):
+            value = value.strip()
+        elif isinstance(value, dict):
+            value = dict((k, self._resolve(v)) for (k, v) in value.items())
+        elif isinstance(value, list):
+            value = list(map(self._resolve, value))
+        return value
+
+    def _lookup(self, path):
+        # fetch the possibly nested value from last_response
+        value = self.last_response
+        if path == "$body":
+            return value
+        path = path.replace(r"\.", "\1")
+        for step in path.split("."):
+            if not step:
+                continue
+            step = step.replace("\1", ".")
+            step = self._resolve(step)
+
+            if (
+                isinstance(step, string_types)
+                and step.isdigit()
+                and isinstance(value, list)
+            ):
+                step = int(step)
+                assert isinstance(value, list)
+                assert len(value) > step
+            elif step == "_arbitrary_key_":
+                return list(value.keys())[0]
+            else:
+                assert step in value
+            value = value[step]
+        return value
+
+    def _feature_enabled(self, name):
+        return False
+
+    def _assert_match_equals(self, a, b):
+        # Handle for large floating points with 'E'
+        if isinstance(b, string_types) and isinstance(a, float) and "e" in repr(a):
+            a = repr(a).replace("e+", "E")
+
+        assert a == b, "%r does not match %r" % (a, b)
+
+
+@pytest.fixture(scope="function")
+def sync_runner(sync_client):
+    return YamlRunner(sync_client)
+
+
+YAML_TEST_SPECS = []
+
+# Try loading the REST API test specs from the Elastic Artifacts API
+try:
+    # Construct the HTTP and OpenSearch client
+    http = urllib3.PoolManager(retries=10)
+    client = get_client()
+
+    # Make a request to OpenSearch for the build hash, we'll be looking for
+    # an artifact with this same hash to download test specs for.
+    client_info = client.info()
+    version_number = client_info["version"]["number"]
+    build_hash = client_info["version"]["build_hash"]
+
+    # Now talk to the artifacts API with the 'STACK_VERSION' environment variable
+    resp = http.request(
+        "GET",
+        "https://artifacts-api.elastic.co/v1/versions/%s" % (version_number,),
+    )
+    resp = json.loads(resp.data.decode("utf-8"))
+
+    # Look through every build and see if one matches the commit hash
+    # we're looking for. If not it's okay, we'll just use the latest and
+    # hope for the best!
+    builds = resp["version"]["builds"]
+    for build in builds:
+        if build["projects"]["opensearch"]["commit_hash"] == build_hash:
+            break
+    else:
+        build = builds[0]  # Use the latest
+
+    # Now we're looking for the 'rest-api-spec-<VERSION>-sources.jar' file
+    # to download and extract in-memory.
+    packages = build["projects"]["opensearch"]["packages"]
+    for package in packages:
+        if re.match(r"rest-resources-zip-.*\.zip", package):
+            package_url = packages[package]["url"]
+            break
+    else:
+        raise RuntimeError(
+            "Could not find the package 'rest-resources-zip-*.zip' in build %r" % build
+        )
+
+    # Download the zip and start reading YAML from the files in memory
+    package_zip = zipfile.ZipFile(io.BytesIO(http.request("GET", package_url).data))
+    for yaml_file in package_zip.namelist():
+        if not re.match(r"^rest-api-spec/test/.*\.ya?ml$", yaml_file):
+            continue
+        yaml_tests = list(yaml.safe_load_all(package_zip.read(yaml_file)))
+
+        # Each file may have a "test" named 'setup' or 'teardown',
+        # these sets of steps should be run at the beginning and end
+        # of every other test within the file so we do one pass to capture those.
+        setup_steps = teardown_steps = None
+        test_numbers_and_steps = []
+        test_number = 0
+
+        for yaml_test in yaml_tests:
+            test_name, test_step = yaml_test.popitem()
+            if test_name == "setup":
+                setup_steps = test_step
+            elif test_name == "teardown":
+                teardown_steps = test_step
+            else:
+                test_numbers_and_steps.append((test_number, test_step))
+                test_number += 1
+
+        # Now we combine setup, teardown, and test_steps into
+        # a set of pytest.param() instances
+        for test_number, test_step in test_numbers_and_steps:
+            # Build the id from the name of the YAML file and
+            # the number within that file. Most important step
+            # is to remove most of the file path prefixes and
+            # the .yml suffix.
+            pytest_test_name = yaml_file.rpartition(".")[0].replace(".", "/")
+            for prefix in ("rest-api-spec/", "test/", "oss/"):
+                if pytest_test_name.startswith(prefix):
+                    pytest_test_name = pytest_test_name[len(prefix) :]
+            pytest_param_id = "%s[%d]" % (pytest_test_name, test_number)
+
+            pytest_param = {
+                "setup": setup_steps,
+                "run": test_step,
+                "teardown": teardown_steps,
+            }
+            # Skip either 'test_name' or 'test_name[x]'
+            if pytest_test_name in SKIP_TESTS or pytest_param_id in SKIP_TESTS:
+                pytest_param["skip"] = True
+
+            YAML_TEST_SPECS.append(pytest.param(pytest_param, id=pytest_param_id))
+
+except Exception as e:
+    warnings.warn("Could not load REST API tests: %s" % (str(e),))
+
+
+if not RUN_ASYNC_REST_API_TESTS:
+
+    @pytest.mark.parametrize("test_spec", YAML_TEST_SPECS)
+    def test_rest_api_spec(test_spec, sync_runner):
+        if test_spec.get("skip", False):
+            pytest.skip("Manually skipped in 'SKIP_TESTS'")
+        sync_runner.use_spec(test_spec)
+        sync_runner.run()

test_opensearchpy/test_server_secured/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.

test_opensearchpy/test_server_secured/test_clients.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+from unittest import TestCase
+
+from opensearchpy import OpenSearch
+from opensearchpy.helpers.test import OPENSEARCH_URL
+
+
+class TestSecurity(TestCase):
+    def test_security(self):
+        client = OpenSearch(
+            OPENSEARCH_URL,
+            http_auth=("admin", "admin"),
+            verify_certs=False,
+        )
+
+        info = client.info()
+        self.assertNotEqual(info["version"]["number"], "")
+        self.assertNotEqual(info["tagline"], "")
+        self.assertTrue(
+            "build_flavor" in info["version"] or "distribution" in info["version"]
+        )

test_opensearchpy/test_transport.py

@@ -0,0 +1,407 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from __future__ import unicode_literals
+
+import json
+import time
+
+from mock import patch
+
+from opensearchpy.connection import Connection
+from opensearchpy.connection_pool import DummyConnectionPool
+from opensearchpy.exceptions import ConnectionError, TransportError
+from opensearchpy.transport import Transport, get_host_info
+
+from .test_cases import TestCase
+
+
+class DummyConnection(Connection):
+    def __init__(self, **kwargs):
+        self.exception = kwargs.pop("exception", None)
+        self.status, self.data = kwargs.pop("status", 200), kwargs.pop("data", "{}")
+        self.headers = kwargs.pop("headers", {})
+        self.calls = []
+        super(DummyConnection, self).__init__(**kwargs)
+
+    def perform_request(self, *args, **kwargs):
+        self.calls.append((args, kwargs))
+        if self.exception:
+            raise self.exception
+        return self.status, self.headers, self.data
+
+
+CLUSTER_NODES = """{
+  "_nodes" : {
+    "total" : 1,
+    "successful" : 1,
+    "failed" : 0
+  },
+  "cluster_name" : "opensearch",
+  "nodes" : {
+    "SRZpKFZdQguhhvifmN6UVA" : {
+      "name" : "SRZpKFZ",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1",
+      "version" : "5.0.0",
+      "build_hash" : "253032b",
+      "roles" : [ "master", "data", "ingest" ],
+      "http" : {
+        "bound_address" : [ "[fe80::1]:9200", "[::1]:9200", "127.0.0.1:9200" ],
+        "publish_address" : "1.1.1.1:123",
+        "max_content_length_in_bytes" : 104857600
+      }
+    }
+  }
+}"""
+
+CLUSTER_NODES_7x_PUBLISH_HOST = """{
+  "_nodes" : {
+    "total" : 1,
+    "successful" : 1,
+    "failed" : 0
+  },
+  "cluster_name" : "opensearch",
+  "nodes" : {
+    "SRZpKFZdQguhhvifmN6UVA" : {
+      "name" : "SRZpKFZ",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1",
+      "version" : "5.0.0",
+      "build_hash" : "253032b",
+      "roles" : [ "master", "data", "ingest" ],
+      "http" : {
+        "bound_address" : [ "[fe80::1]:9200", "[::1]:9200", "127.0.0.1:9200" ],
+        "publish_address" : "somehost.tld/1.1.1.1:123",
+        "max_content_length_in_bytes" : 104857600
+      }
+    }
+  }
+}"""
+
+
+class TestHostsInfoCallback(TestCase):
+    def test_master_only_nodes_are_ignored(self):
+        nodes = [
+            {"roles": ["master"]},
+            {"roles": ["master", "data", "ingest"]},
+            {"roles": ["data", "ingest"]},
+            {"roles": []},
+            {},
+        ]
+        chosen = [
+            i
+            for i, node_info in enumerate(nodes)
+            if get_host_info(node_info, i) is not None
+        ]
+        self.assertEqual([1, 2, 3, 4], chosen)
+
+
+class TestTransport(TestCase):
+    def test_single_connection_uses_dummy_connection_pool(self):
+        t = Transport([{}])
+        self.assertIsInstance(t.connection_pool, DummyConnectionPool)
+        t = Transport([{"host": "localhost"}])
+        self.assertIsInstance(t.connection_pool, DummyConnectionPool)
+
+    def test_request_timeout_extracted_from_params_and_passed(self):
+        t = Transport([{}], connection_class=DummyConnection)
+
+        t.perform_request("GET", "/", params={"request_timeout": 42})
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(("GET", "/", {}, None), t.get_connection().calls[0][0])
+        self.assertEqual(
+            {"timeout": 42, "ignore": (), "headers": None},
+            t.get_connection().calls[0][1],
+        )
+
+    def test_opaque_id(self):
+        t = Transport([{}], opaque_id="app-1", connection_class=DummyConnection)
+
+        t.perform_request("GET", "/")
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(("GET", "/", None, None), t.get_connection().calls[0][0])
+        self.assertEqual(
+            {"timeout": None, "ignore": (), "headers": None},
+            t.get_connection().calls[0][1],
+        )
+
+        # Now try with an 'x-opaque-id' set on perform_request().
+        t.perform_request("GET", "/", headers={"x-opaque-id": "request-1"})
+        self.assertEqual(2, len(t.get_connection().calls))
+        self.assertEqual(("GET", "/", None, None), t.get_connection().calls[1][0])
+        self.assertEqual(
+            {"timeout": None, "ignore": (), "headers": {"x-opaque-id": "request-1"}},
+            t.get_connection().calls[1][1],
+        )
+
+    def test_request_with_custom_user_agent_header(self):
+        t = Transport([{}], connection_class=DummyConnection)
+
+        t.perform_request("GET", "/", headers={"user-agent": "my-custom-value/1.2.3"})
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(
+            {
+                "timeout": None,
+                "ignore": (),
+                "headers": {"user-agent": "my-custom-value/1.2.3"},
+            },
+            t.get_connection().calls[0][1],
+        )
+
+    def test_send_get_body_as_source(self):
+        t = Transport([{}], send_get_body_as="source", connection_class=DummyConnection)
+
+        t.perform_request("GET", "/", body={})
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(
+            ("GET", "/", {"source": "{}"}, None), t.get_connection().calls[0][0]
+        )
+
+    def test_send_get_body_as_post(self):
+        t = Transport([{}], send_get_body_as="POST", connection_class=DummyConnection)
+
+        t.perform_request("GET", "/", body={})
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(("POST", "/", None, b"{}"), t.get_connection().calls[0][0])
+
+    def test_body_gets_encoded_into_bytes(self):
+        t = Transport([{}], connection_class=DummyConnection)
+
+        t.perform_request("GET", "/", body="你好")
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(
+            ("GET", "/", None, b"\xe4\xbd\xa0\xe5\xa5\xbd"),
+            t.get_connection().calls[0][0],
+        )
+
+    def test_body_bytes_get_passed_untouched(self):
+        t = Transport([{}], connection_class=DummyConnection)
+
+        body = b"\xe4\xbd\xa0\xe5\xa5\xbd"
+        t.perform_request("GET", "/", body=body)
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(("GET", "/", None, body), t.get_connection().calls[0][0])
+
+    def test_body_surrogates_replaced_encoded_into_bytes(self):
+        t = Transport([{}], connection_class=DummyConnection)
+
+        t.perform_request("GET", "/", body="你好\uda6a")
+        self.assertEqual(1, len(t.get_connection().calls))
+        self.assertEqual(
+            ("GET", "/", None, b"\xe4\xbd\xa0\xe5\xa5\xbd\xed\xa9\xaa"),
+            t.get_connection().calls[0][0],
+        )
+
+    def test_kwargs_passed_on_to_connections(self):
+        t = Transport([{"host": "google.com"}], port=123)
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertEqual("http://google.com:123", t.connection_pool.connections[0].host)
+
+    def test_kwargs_passed_on_to_connection_pool(self):
+        dt = object()
+        t = Transport([{}, {}], dead_timeout=dt)
+        self.assertIs(dt, t.connection_pool.dead_timeout)
+
+    def test_custom_connection_class(self):
+        class MyConnection(object):
+            def __init__(self, **kwargs):
+                self.kwargs = kwargs
+
+        t = Transport([{}], connection_class=MyConnection)
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertIsInstance(t.connection_pool.connections[0], MyConnection)
+
+    def test_add_connection(self):
+        t = Transport([{}], randomize_hosts=False)
+        t.add_connection({"host": "google.com", "port": 1234})
+
+        self.assertEqual(2, len(t.connection_pool.connections))
+        self.assertEqual(
+            "http://google.com:1234", t.connection_pool.connections[1].host
+        )
+
+    def test_request_will_fail_after_X_retries(self):
+        t = Transport(
+            [{"exception": ConnectionError("abandon ship")}],
+            connection_class=DummyConnection,
+        )
+
+        self.assertRaises(ConnectionError, t.perform_request, "GET", "/")
+        self.assertEqual(4, len(t.get_connection().calls))
+
+    def test_failed_connection_will_be_marked_as_dead(self):
+        t = Transport(
+            [{"exception": ConnectionError("abandon ship")}] * 2,
+            connection_class=DummyConnection,
+        )
+
+        self.assertRaises(ConnectionError, t.perform_request, "GET", "/")
+        self.assertEqual(0, len(t.connection_pool.connections))
+
+    def test_resurrected_connection_will_be_marked_as_live_on_success(self):
+        for method in ("GET", "HEAD"):
+            t = Transport([{}, {}], connection_class=DummyConnection)
+            con1 = t.connection_pool.get_connection()
+            con2 = t.connection_pool.get_connection()
+            t.connection_pool.mark_dead(con1)
+            t.connection_pool.mark_dead(con2)
+
+            t.perform_request(method, "/")
+            self.assertEqual(1, len(t.connection_pool.connections))
+            self.assertEqual(1, len(t.connection_pool.dead_count))
+
+    def test_sniff_will_use_seed_connections(self):
+        t = Transport([{"data": CLUSTER_NODES}], connection_class=DummyConnection)
+        t.set_connections([{"data": "invalid"}])
+
+        t.sniff_hosts()
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertEqual("http://1.1.1.1:123", t.get_connection().host)
+
+    def test_sniff_on_start_fetches_and_uses_nodes_list(self):
+        t = Transport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_start=True,
+        )
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertEqual("http://1.1.1.1:123", t.get_connection().host)
+
+    def test_sniff_on_start_ignores_sniff_timeout(self):
+        t = Transport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_start=True,
+            sniff_timeout=12,
+        )
+        self.assertEqual(
+            (("GET", "/_nodes/_all/http"), {"timeout": None}),
+            t.seed_connections[0].calls[0],
+        )
+
+    def test_sniff_uses_sniff_timeout(self):
+        t = Transport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_timeout=42,
+        )
+        t.sniff_hosts()
+        self.assertEqual(
+            (("GET", "/_nodes/_all/http"), {"timeout": 42}),
+            t.seed_connections[0].calls[0],
+        )
+
+    def test_sniff_reuses_connection_instances_if_possible(self):
+        t = Transport(
+            [{"data": CLUSTER_NODES}, {"host": "1.1.1.1", "port": 123}],
+            connection_class=DummyConnection,
+            randomize_hosts=False,
+        )
+        connection = t.connection_pool.connections[1]
+
+        t.sniff_hosts()
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertIs(connection, t.get_connection())
+
+    def test_sniff_on_fail_triggers_sniffing_on_fail(self):
+        t = Transport(
+            [{"exception": ConnectionError("abandon ship")}, {"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_connection_fail=True,
+            max_retries=0,
+            randomize_hosts=False,
+        )
+
+        self.assertRaises(ConnectionError, t.perform_request, "GET", "/")
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertEqual("http://1.1.1.1:123", t.get_connection().host)
+
+    @patch("opensearchpy.transport.Transport.sniff_hosts")
+    def test_sniff_on_fail_failing_does_not_prevent_retires(self, sniff_hosts):
+        sniff_hosts.side_effect = [TransportError("sniff failed")]
+        t = Transport(
+            [{"exception": ConnectionError("abandon ship")}, {"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_connection_fail=True,
+            max_retries=3,
+            randomize_hosts=False,
+        )
+
+        conn_err, conn_data = t.connection_pool.connections
+        response = t.perform_request("GET", "/")
+        self.assertEqual(json.loads(CLUSTER_NODES), response)
+        self.assertEqual(1, sniff_hosts.call_count)
+        self.assertEqual(1, len(conn_err.calls))
+        self.assertEqual(1, len(conn_data.calls))
+
+    def test_sniff_after_n_seconds(self):
+        t = Transport(
+            [{"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniffer_timeout=5,
+        )
+
+        for _ in range(4):
+            t.perform_request("GET", "/")
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertIsInstance(t.get_connection(), DummyConnection)
+        t.last_sniff = time.time() - 5.1
+
+        t.perform_request("GET", "/")
+        self.assertEqual(1, len(t.connection_pool.connections))
+        self.assertEqual("http://1.1.1.1:123", t.get_connection().host)
+        self.assertTrue(time.time() - 1 < t.last_sniff < time.time() + 0.01)
+
+    def test_sniff_7x_publish_host(self):
+        # Test the response shaped when a 7.x node has publish_host set
+        # and the returend data is shaped in the fqdn/ip:port format.
+        t = Transport(
+            [{"data": CLUSTER_NODES_7x_PUBLISH_HOST}],
+            connection_class=DummyConnection,
+            sniff_timeout=42,
+        )
+        t.sniff_hosts()
+        # Ensure we parsed out the fqdn and port from the fqdn/ip:port string.
+        self.assertEqual(
+            t.connection_pool.connection_opts[0][1],
+            {"host": "somehost.tld", "port": 123},
+        )
+
+    @patch("opensearchpy.transport.Transport.sniff_hosts")
+    def test_sniffing_disabled_on_cloud_instances(self, sniff_hosts):
+        t = Transport(
+            [{}],
+            sniff_on_start=True,
+            sniff_on_connection_fail=True,
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+        )
+
+        self.assertFalse(t.sniff_on_connection_fail)
+        self.assertIs(sniff_hosts.call_args, None)  # Assert not called.

test_opensearchpy/test_types/README.md

@@ -0,0 +1,6 @@
+# Type Hints
+
+All of these scripts are used to test the type hinting
+distributed with the `opensearch` package.
+These scripts simulate normal usage of the client and are run
+through `mypy --strict` as a part of continuous integration.

test_opensearchpy/test_types/aliased_types.py

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from typing import Any, AsyncGenerator, Dict, Generator
+
+from opensearchpy1 import (
+    AIOHttpConnection,
+    AsyncOpenSearch,
+    AsyncTransport,
+    ConnectionPool,
+    OpenSearch,
+    RequestsHttpConnection,
+    Transport,
+)
+from opensearchpy1.helpers import (
+    async_bulk,
+    async_reindex,
+    async_scan,
+    async_streaming_bulk,
+    bulk,
+    reindex,
+    scan,
+    streaming_bulk,
+)
+
+client = OpenSearch(
+    [{"host": "localhost", "port": 9443}],
+    transport_class=Transport,
+)
+t = Transport(
+    [{}],
+    connection_class=RequestsHttpConnection,
+    connection_pool_class=ConnectionPool,
+    sniff_on_start=True,
+    sniffer_timeout=0.1,
+    sniff_timeout=1,
+    sniff_on_connection_fail=False,
+    max_retries=1,
+    retry_on_status={100, 400, 503},
+    retry_on_timeout=True,
+    send_get_body_as="source",
+)
+
+
+def sync_gen() -> Generator[Dict[Any, Any], None, None]:
+    yield {}
+
+
+def scan_types() -> None:
+    for _ in scan(
+        client,
+        query={"query": {"match_all": {}}},
+        request_timeout=10,
+        clear_scroll=True,
+        scroll_kwargs={"request_timeout": 10},
+    ):
+        pass
+    for _ in scan(
+        client,
+        raise_on_error=False,
+        preserve_order=False,
+        scroll="10m",
+        size=10,
+        request_timeout=10.0,
+    ):
+        pass
+
+
+def streaming_bulk_types() -> None:
+    for _ in streaming_bulk(client, sync_gen()):
+        pass
+    for _ in streaming_bulk(client, sync_gen().__iter__()):
+        pass
+    for _ in streaming_bulk(client, [{}]):
+        pass
+    for _ in streaming_bulk(client, ({},)):
+        pass
+
+
+def bulk_types() -> None:
+    _, _ = bulk(client, sync_gen())
+    _, _ = bulk(client, sync_gen().__iter__())
+    _, _ = bulk(client, [{}])
+    _, _ = bulk(client, ({},))
+
+
+def reindex_types() -> None:
+    _, _ = reindex(
+        client, "src-index", "target-index", query={"query": {"match": {"key": "val"}}}
+    )
+    _, _ = reindex(
+        client,
+        source_index="src-index",
+        target_index="target-index",
+        target_client=client,
+    )
+    _, _ = reindex(
+        client,
+        "src-index",
+        "target-index",
+        chunk_size=1,
+        scroll="10m",
+        scan_kwargs={"request_timeout": 10},
+        bulk_kwargs={"request_timeout": 10},
+    )
+
+
+client2 = AsyncOpenSearch(
+    [{"host": "localhost", "port": 9443}],
+    transport_class=AsyncTransport,
+)
+t2 = AsyncTransport(
+    [{}],
+    connection_class=AIOHttpConnection,
+    connection_pool_class=ConnectionPool,
+    sniff_on_start=True,
+    sniffer_timeout=0.1,
+    sniff_timeout=1,
+    sniff_on_connection_fail=False,
+    max_retries=1,
+    retry_on_status={100, 400, 503},
+    retry_on_timeout=True,
+    send_get_body_as="source",
+)
+
+
+async def async_gen() -> AsyncGenerator[Dict[Any, Any], None]:
+    yield {}
+
+
+async def async_scan_types() -> None:
+    async for _ in async_scan(
+        client2,
+        query={"query": {"match_all": {}}},
+        request_timeout=10,
+        clear_scroll=True,
+        scroll_kwargs={"request_timeout": 10},
+    ):
+        pass
+    async for _ in async_scan(
+        client2,
+        raise_on_error=False,
+        preserve_order=False,
+        scroll="10m",
+        size=10,
+        request_timeout=10.0,
+    ):
+        pass
+
+
+async def async_streaming_bulk_types() -> None:
+    async for _ in async_streaming_bulk(client2, async_gen()):
+        pass
+    async for _ in async_streaming_bulk(client2, async_gen().__aiter__()):
+        pass
+    async for _ in async_streaming_bulk(client2, [{}]):
+        pass
+    async for _ in async_streaming_bulk(client2, ({},)):
+        pass
+
+
+async def async_bulk_types() -> None:
+    _, _ = await async_bulk(client2, async_gen())
+    _, _ = await async_bulk(client2, async_gen().__aiter__())
+    _, _ = await async_bulk(client2, [{}])
+    _, _ = await async_bulk(client2, ({},))
+
+
+async def async_reindex_types() -> None:
+    _, _ = await async_reindex(
+        client2, "src-index", "target-index", query={"query": {"match": {"key": "val"}}}
+    )
+    _, _ = await async_reindex(
+        client2,
+        source_index="src-index",
+        target_index="target-index",
+        target_client=client2,
+    )
+    _, _ = await async_reindex(
+        client2,
+        "src-index",
+        "target-index",
+        chunk_size=1,
+        scroll="10m",
+        scan_kwargs={"request_timeout": 10},
+        bulk_kwargs={"request_timeout": 10},
+    )

test_opensearchpy/test_types/async_types.py

@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from typing import Any, AsyncGenerator, Dict
+
+from opensearchpy import (
+    AIOHttpConnection,
+    AsyncOpenSearch,
+    AsyncTransport,
+    ConnectionPool,
+)
+from opensearchpy.helpers import (
+    async_bulk,
+    async_reindex,
+    async_scan,
+    async_streaming_bulk,
+)
+
+client = AsyncOpenSearch(
+    [{"host": "localhost", "port": 9443}],
+    transport_class=AsyncTransport,
+)
+t = AsyncTransport(
+    [{}],
+    connection_class=AIOHttpConnection,
+    connection_pool_class=ConnectionPool,
+    sniff_on_start=True,
+    sniffer_timeout=0.1,
+    sniff_timeout=1,
+    sniff_on_connection_fail=False,
+    max_retries=1,
+    retry_on_status={100, 400, 503},
+    retry_on_timeout=True,
+    send_get_body_as="source",
+)
+
+
+async def async_gen() -> AsyncGenerator[Dict[Any, Any], None]:
+    yield {}
+
+
+async def async_scan_types() -> None:
+    async for _ in async_scan(
+        client,
+        query={"query": {"match_all": {}}},
+        request_timeout=10,
+        clear_scroll=True,
+        scroll_kwargs={"request_timeout": 10},
+    ):
+        pass
+    async for _ in async_scan(
+        client,
+        raise_on_error=False,
+        preserve_order=False,
+        scroll="10m",
+        size=10,
+        request_timeout=10.0,
+    ):
+        pass
+
+
+async def async_streaming_bulk_types() -> None:
+    async for _ in async_streaming_bulk(client, async_gen()):
+        pass
+    async for _ in async_streaming_bulk(client, async_gen().__aiter__()):
+        pass
+    async for _ in async_streaming_bulk(client, [{}]):
+        pass
+    async for _ in async_streaming_bulk(client, ({},)):
+        pass
+
+
+async def async_bulk_types() -> None:
+    _, _ = await async_bulk(client, async_gen())
+    _, _ = await async_bulk(client, async_gen().__aiter__())
+    _, _ = await async_bulk(client, [{}])
+    _, _ = await async_bulk(client, ({},))
+
+
+async def async_reindex_types() -> None:
+    _, _ = await async_reindex(
+        client, "src-index", "target-index", query={"query": {"match": {"key": "val"}}}
+    )
+    _, _ = await async_reindex(
+        client,
+        source_index="src-index",
+        target_index="target-index",
+        target_client=client,
+    )
+    _, _ = await async_reindex(
+        client,
+        "src-index",
+        "target-index",
+        chunk_size=1,
+        scroll="10m",
+        scan_kwargs={"request_timeout": 10},
+        bulk_kwargs={"request_timeout": 10},
+    )

test_opensearchpy/test_types/sync_types.py

@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+from typing import Any, Dict, Generator
+
+from opensearchpy import ConnectionPool, OpenSearch, RequestsHttpConnection, Transport
+from opensearchpy.helpers import bulk, reindex, scan, streaming_bulk
+
+client = OpenSearch(
+    [{"host": "localhost", "port": 9443}],
+    transport_class=Transport,
+)
+t = Transport(
+    [{}],
+    connection_class=RequestsHttpConnection,
+    connection_pool_class=ConnectionPool,
+    sniff_on_start=True,
+    sniffer_timeout=0.1,
+    sniff_timeout=1,
+    sniff_on_connection_fail=False,
+    max_retries=1,
+    retry_on_status={100, 400, 503},
+    retry_on_timeout=True,
+    send_get_body_as="source",
+)
+
+
+def sync_gen() -> Generator[Dict[Any, Any], None, None]:
+    yield {}
+
+
+def scan_types() -> None:
+    for _ in scan(
+        client,
+        query={"query": {"match_all": {}}},
+        request_timeout=10,
+        clear_scroll=True,
+        scroll_kwargs={"request_timeout": 10},
+    ):
+        pass
+    for _ in scan(
+        client,
+        raise_on_error=False,
+        preserve_order=False,
+        scroll="10m",
+        size=10,
+        request_timeout=10.0,
+    ):
+        pass
+
+
+def streaming_bulk_types() -> None:
+    for _ in streaming_bulk(client, sync_gen()):
+        pass
+    for _ in streaming_bulk(client, sync_gen().__iter__()):
+        pass
+    for _ in streaming_bulk(client, [{}]):
+        pass
+    for _ in streaming_bulk(client, ({},)):
+        pass
+
+
+def bulk_types() -> None:
+    _, _ = bulk(client, sync_gen())
+    _, _ = bulk(client, sync_gen().__iter__())
+    _, _ = bulk(client, [{}])
+    _, _ = bulk(client, ({},))
+
+
+def reindex_types() -> None:
+    _, _ = reindex(
+        client, "src-index", "target-index", query={"query": {"match": {"key": "val"}}}
+    )
+    _, _ = reindex(
+        client,
+        source_index="src-index",
+        target_index="target-index",
+        target_client=client,
+    )
+    _, _ = reindex(
+        client,
+        "src-index",
+        "target-index",
+        chunk_size=1,
+        scroll="10m",
+        scan_kwargs={"request_timeout": 10},
+        bulk_kwargs={"request_timeout": 10},
+    )

test_opensearchpy/utils.py

@@ -0,0 +1,176 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import time
+
+from opensearchpy import OpenSearch
+
+
+def wipe_cluster(client):
+    """Wipes a cluster clean between test cases"""
+    close_after_wipe = False
+    try:
+        # If client is async we need to replace the client
+        # with a synchronous one.
+        from opensearchpy import AsyncOpenSearch
+
+        if isinstance(client, AsyncOpenSearch):
+            client = OpenSearch(client.transport.hosts, verify_certs=False)
+            close_after_wipe = True
+    except ImportError:
+        pass
+
+    wipe_snapshots(client)
+    wipe_indices(client)
+
+    client.indices.delete_template(name="*")
+    client.indices.delete_index_template(name="*")
+    client.cluster.delete_component_template(name="*")
+
+    wipe_cluster_settings(client)
+
+    wait_for_cluster_state_updates_to_finish(client)
+    if close_after_wipe:
+        client.close()
+
+
+def wipe_cluster_settings(client):
+    settings = client.cluster.get_settings()
+    new_settings = {}
+    for name, value in settings.items():
+        if value:
+            new_settings.setdefault(name, {})
+            for key in value.keys():
+                new_settings[name][key + ".*"] = None
+    if new_settings:
+        client.cluster.put_settings(body=new_settings)
+
+
+def wipe_snapshots(client):
+    """Deletes all the snapshots and repositories from the cluster"""
+    in_progress_snapshots = []
+
+    repos = client.snapshot.get_repository(repository="_all")
+    for repo_name, repo in repos.items():
+        if repo["type"] == "fs":
+            snapshots = client.snapshot.get(
+                repository=repo_name, snapshot="_all", ignore_unavailable=True
+            )
+            for snapshot in snapshots["snapshots"]:
+                if snapshot["state"] == "IN_PROGRESS":
+                    in_progress_snapshots.append(snapshot)
+                else:
+                    client.snapshot.delete(
+                        repository=repo_name,
+                        snapshot=snapshot["snapshot"],
+                        ignore=404,
+                    )
+
+        client.snapshot.delete_repository(repository=repo_name, ignore=404)
+
+    assert in_progress_snapshots == []
+
+
+def wipe_data_streams(client):
+    try:
+        client.indices.delete_data_stream(name="*", expand_wildcards="all")
+    except Exception:
+        client.indices.delete_data_stream(name="*")
+
+
+def wipe_indices(client):
+    client.indices.delete(
+        index="*,-.ds-ilm-history-*",
+        expand_wildcards="all",
+        ignore=404,
+    )
+
+
+def wipe_searchable_snapshot_indices(client):
+    cluster_metadata = client.cluster.state(
+        metric="metadata",
+        filter_path="metadata.indices.*.settings.index.store.snapshot",
+    )
+    if cluster_metadata:
+        for index in cluster_metadata["metadata"]["indices"].keys():
+            client.indices.delete(index=index)
+
+
+def wipe_slm_policies(client):
+    for policy in client.slm.get_lifecycle():
+        client.slm.delete_lifecycle(policy_id=policy["name"])
+
+
+def wipe_auto_follow_patterns(client):
+    for pattern in client.ccr.get_auto_follow_pattern()["patterns"]:
+        client.ccr.delete_auto_follow_pattern(name=pattern["name"])
+
+
+def wipe_node_shutdown_metadata(client):
+    shutdown_status = client.shutdown.get_node()
+    # If response contains these two keys the feature flag isn't enabled
+    # on this cluster so skip this step now.
+    if "_nodes" in shutdown_status and "cluster_name" in shutdown_status:
+        return
+
+    for shutdown_node in shutdown_status.get("nodes", []):
+        node_id = shutdown_node["node_id"]
+        client.shutdown.delete_node(node_id=node_id)
+
+
+def wipe_tasks(client):
+    tasks = client.tasks.list()
+    for node_name, node in tasks.get("node", {}).items():
+        for task_id in node.get("tasks", ()):
+            client.tasks.cancel(task_id=task_id, wait_for_completion=True)
+
+
+def wait_for_pending_tasks(client, filter, timeout=30):
+    end_time = time.time() + timeout
+    while time.time() < end_time:
+        tasks = client.cat.tasks(detailed=True).split("\n")
+        if not any(filter in task for task in tasks):
+            break
+
+
+def wait_for_pending_datafeeds_and_jobs(client, timeout=30):
+    end_time = time.time() + timeout
+    while time.time() < end_time:
+        if (
+            client.ml.get_datafeeds(datafeed_id="*", allow_no_datafeeds=True)["count"]
+            == 0
+        ):
+            break
+    while time.time() < end_time:
+        if client.ml.get_jobs(job_id="*", allow_no_jobs=True)["count"] == 0:
+            break
+
+
+def wait_for_cluster_state_updates_to_finish(client, timeout=30):
+    end_time = time.time() + timeout
+    while time.time() < end_time:
+        if not client.cluster.pending_tasks().get("tasks", ()):
+            break