Rename module to opensearchpy
To avoid conflict with an existing package by name 'opensearch' being present Signed-off-by: Rushi Agrawal <[email protected]>
This commit is contained in:
Rushi Agrawal
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
#
|
||||
# Licensed to Elasticsearch B.V. under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch B.V. licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from unittest import SkipTest
|
||||
|
||||
from opensearchpy.helpers import test
|
||||
from opensearchpy.helpers.test import OpenSearchTestCase as BaseTestCase
|
||||
|
||||
client = None
|
||||
|
||||
|
||||
def get_client(**kwargs):
|
||||
global client
|
||||
if client is False:
|
||||
raise SkipTest("No client is available")
|
||||
if client is not None and not kwargs:
|
||||
return client
|
||||
|
||||
# try and locate manual override in the local environment
|
||||
try:
|
||||
from test_opensearchpy.local import get_client as local_get_client
|
||||
|
||||
new_client = local_get_client(**kwargs)
|
||||
except ImportError:
|
||||
# fallback to using vanilla client
|
||||
try:
|
||||
new_client = test.get_test_client(**kwargs)
|
||||
except SkipTest:
|
||||
client = False
|
||||
raise
|
||||
|
||||
if not kwargs:
|
||||
client = new_client
|
||||
|
||||
return new_client
|
||||
|
||||
|
||||
def setup_module():
|
||||
get_client()
|
||||
|
||||
|
||||
class OpenSearchTestCase(BaseTestCase):
|
||||
@staticmethod
|
||||
def _get_client(**kwargs):
|
||||
return get_client(**kwargs)
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
#
|
||||
# Licensed to Elasticsearch B.V. under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch B.V. licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import opensearchpy
|
||||
from opensearchpy.helpers.test import CA_CERTS, OPENSEARCH_URL
|
||||
|
||||
from ..utils import wipe_cluster
|
||||
|
||||
# Information about the OpenSearch instance running, if any
|
||||
# Used for
|
||||
OPENSEARCH_VERSION = ""
|
||||
OPENSEARCH_BUILD_HASH = ""
|
||||
OPENSEARCH_REST_API_TESTS = []
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def sync_client_factory():
|
||||
client = None
|
||||
try:
|
||||
# Configure the client with certificates and optionally
|
||||
# an HTTP conn class depending on 'PYTHON_CONNECTION_CLASS' envvar
|
||||
kw = {
|
||||
"timeout": 3,
|
||||
"ca_certs": CA_CERTS,
|
||||
"headers": {"Authorization": "Basic ZWxhc3RpYzpjaGFuZ2VtZQ=="},
|
||||
}
|
||||
if "PYTHON_CONNECTION_CLASS" in os.environ:
|
||||
from opensearchpy import connection
|
||||
|
||||
kw["connection_class"] = getattr(
|
||||
connection, os.environ["PYTHON_CONNECTION_CLASS"]
|
||||
)
|
||||
|
||||
# We do this little dance with the URL to force
|
||||
# Requests to respect 'headers: None' within rest API spec tests.
|
||||
client = opensearchpy.OpenSearch(
|
||||
OPENSEARCH_URL.replace("elastic:changeme@", ""), **kw
|
||||
)
|
||||
|
||||
# Wait for the cluster to report a status of 'yellow'
|
||||
for _ in range(100):
|
||||
try:
|
||||
client.cluster.health(wait_for_status="yellow")
|
||||
break
|
||||
except ConnectionError:
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
pytest.skip("OpenSearch wasn't running at %r" % (OPENSEARCH_URL,))
|
||||
|
||||
wipe_cluster(client)
|
||||
yield client
|
||||
finally:
|
||||
if client:
|
||||
client.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def sync_client(sync_client_factory):
|
||||
try:
|
||||
yield sync_client_factory
|
||||
finally:
|
||||
wipe_cluster(sync_client_factory)
|
||||
@@ -0,0 +1,51 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
#
|
||||
# Licensed to Elasticsearch B.V. under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch B.V. licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from . import OpenSearchTestCase
|
||||
|
||||
|
||||
class TestUnicode(OpenSearchTestCase):
|
||||
def test_indices_analyze(self):
|
||||
self.client.indices.analyze(body='{"text": "привет"}')
|
||||
|
||||
|
||||
class TestBulk(OpenSearchTestCase):
|
||||
def test_bulk_works_with_string_body(self):
|
||||
docs = '{ "index" : { "_index" : "bulk_test_index", "_id" : "1" } }\n{"answer": 42}'
|
||||
response = self.client.bulk(body=docs)
|
||||
|
||||
self.assertFalse(response["errors"])
|
||||
self.assertEqual(1, len(response["items"]))
|
||||
|
||||
def test_bulk_works_with_bytestring_body(self):
|
||||
docs = b'{ "index" : { "_index" : "bulk_test_index", "_id" : "2" } }\n{"answer": 42}'
|
||||
response = self.client.bulk(body=docs)
|
||||
|
||||
self.assertFalse(response["errors"])
|
||||
self.assertEqual(1, len(response["items"]))
|
||||
@@ -0,0 +1,775 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
#
|
||||
# Licensed to Elasticsearch B.V. under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch B.V. licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from mock import patch
|
||||
|
||||
from opensearchpy import TransportError, helpers
|
||||
from opensearchpy.helpers import ScanError
|
||||
|
||||
from ..test_cases import SkipTest
|
||||
from . import OpenSearchTestCase
|
||||
|
||||
|
||||
class FailingBulkClient(object):
|
||||
def __init__(
|
||||
self, client, fail_at=(2,), fail_with=TransportError(599, "Error!", {})
|
||||
):
|
||||
self.client = client
|
||||
self._called = 0
|
||||
self._fail_at = fail_at
|
||||
self.transport = client.transport
|
||||
self._fail_with = fail_with
|
||||
|
||||
def bulk(self, *args, **kwargs):
|
||||
self._called += 1
|
||||
if self._called in self._fail_at:
|
||||
raise self._fail_with
|
||||
return self.client.bulk(*args, **kwargs)
|
||||
|
||||
|
||||
class TestStreamingBulk(OpenSearchTestCase):
|
||||
def test_actions_remain_unchanged(self):
|
||||
actions = [{"_id": 1}, {"_id": 2}]
|
||||
for ok, item in helpers.streaming_bulk(
|
||||
self.client, actions, index="test-index"
|
||||
):
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual([{"_id": 1}, {"_id": 2}], actions)
|
||||
|
||||
def test_all_documents_get_inserted(self):
|
||||
docs = [{"answer": x, "_id": x} for x in range(100)]
|
||||
for ok, item in helpers.streaming_bulk(
|
||||
self.client, docs, index="test-index", refresh=True
|
||||
):
|
||||
self.assertTrue(ok)
|
||||
|
||||
self.assertEqual(100, self.client.count(index="test-index")["count"])
|
||||
self.assertEqual(
|
||||
{"answer": 42}, self.client.get(index="test-index", id=42)["_source"]
|
||||
)
|
||||
|
||||
def test_all_errors_from_chunk_are_raised_on_failure(self):
|
||||
self.client.indices.create(
|
||||
"i",
|
||||
{
|
||||
"mappings": {"properties": {"a": {"type": "integer"}}},
|
||||
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
|
||||
},
|
||||
)
|
||||
self.client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
try:
|
||||
for ok, item in helpers.streaming_bulk(
|
||||
self.client, [{"a": "b"}, {"a": "c"}], index="i", raise_on_error=True
|
||||
):
|
||||
self.assertTrue(ok)
|
||||
except helpers.BulkIndexError as e:
|
||||
self.assertEqual(2, len(e.errors))
|
||||
else:
|
||||
assert False, "exception should have been raised"
|
||||
|
||||
def test_different_op_types(self):
|
||||
if self.opensearch_version() < (0, 90, 1):
|
||||
raise SkipTest("update supported since 0.90.1")
|
||||
self.client.index(index="i", id=45, body={})
|
||||
self.client.index(index="i", id=42, body={})
|
||||
docs = [
|
||||
{"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
|
||||
{"_op_type": "delete", "_index": "i", "_type": "_doc", "_id": 45},
|
||||
{
|
||||
"_op_type": "update",
|
||||
"_index": "i",
|
||||
"_type": "_doc",
|
||||
"_id": 42,
|
||||
"doc": {"answer": 42},
|
||||
},
|
||||
]
|
||||
for ok, item in helpers.streaming_bulk(self.client, docs):
|
||||
self.assertTrue(ok)
|
||||
|
||||
self.assertFalse(self.client.exists(index="i", id=45))
|
||||
self.assertEqual({"answer": 42}, self.client.get(index="i", id=42)["_source"])
|
||||
self.assertEqual({"f": "v"}, self.client.get(index="i", id=47)["_source"])
|
||||
|
||||
def test_transport_error_can_becaught(self):
|
||||
failing_client = FailingBulkClient(self.client)
|
||||
docs = [
|
||||
{"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
|
||||
{"_index": "i", "_type": "_doc", "_id": 45, "f": "v"},
|
||||
{"_index": "i", "_type": "_doc", "_id": 42, "f": "v"},
|
||||
]
|
||||
|
||||
results = list(
|
||||
helpers.streaming_bulk(
|
||||
failing_client,
|
||||
docs,
|
||||
raise_on_exception=False,
|
||||
raise_on_error=False,
|
||||
chunk_size=1,
|
||||
)
|
||||
)
|
||||
self.assertEqual(3, len(results))
|
||||
self.assertEqual([True, False, True], [r[0] for r in results])
|
||||
|
||||
exc = results[1][1]["index"].pop("exception")
|
||||
self.assertIsInstance(exc, TransportError)
|
||||
self.assertEqual(599, exc.status_code)
|
||||
self.assertEqual(
|
||||
{
|
||||
"index": {
|
||||
"_index": "i",
|
||||
"_type": "_doc",
|
||||
"_id": 45,
|
||||
"data": {"f": "v"},
|
||||
"error": "TransportError(599, 'Error!')",
|
||||
"status": 599,
|
||||
}
|
||||
},
|
||||
results[1][1],
|
||||
)
|
||||
|
||||
def test_rejected_documents_are_retried(self):
|
||||
failing_client = FailingBulkClient(
|
||||
self.client, fail_with=TransportError(429, "Rejected!", {})
|
||||
)
|
||||
docs = [
|
||||
{"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
|
||||
{"_index": "i", "_type": "_doc", "_id": 45, "f": "v"},
|
||||
{"_index": "i", "_type": "_doc", "_id": 42, "f": "v"},
|
||||
]
|
||||
results = list(
|
||||
helpers.streaming_bulk(
|
||||
failing_client,
|
||||
docs,
|
||||
raise_on_exception=False,
|
||||
raise_on_error=False,
|
||||
chunk_size=1,
|
||||
max_retries=1,
|
||||
initial_backoff=0,
|
||||
)
|
||||
)
|
||||
self.assertEqual(3, len(results))
|
||||
self.assertEqual([True, True, True], [r[0] for r in results])
|
||||
self.client.indices.refresh(index="i")
|
||||
res = self.client.search(index="i")
|
||||
self.assertEqual({"value": 3, "relation": "eq"}, res["hits"]["total"])
|
||||
self.assertEqual(4, failing_client._called)
|
||||
|
||||
def test_rejected_documents_are_retried_at_most_max_retries_times(self):
|
||||
failing_client = FailingBulkClient(
|
||||
self.client, fail_at=(1, 2), fail_with=TransportError(429, "Rejected!", {})
|
||||
)
|
||||
|
||||
docs = [
|
||||
{"_index": "i", "_type": "_doc", "_id": 47, "f": "v"},
|
||||
{"_index": "i", "_type": "_doc", "_id": 45, "f": "v"},
|
||||
{"_index": "i", "_type": "_doc", "_id": 42, "f": "v"},
|
||||
]
|
||||
results = list(
|
||||
helpers.streaming_bulk(
|
||||
failing_client,
|
||||
docs,
|
||||
raise_on_exception=False,
|
||||
raise_on_error=False,
|
||||
chunk_size=1,
|
||||
max_retries=1,
|
||||
initial_backoff=0,
|
||||
)
|
||||
)
|
||||
self.assertEqual(3, len(results))
|
||||
self.assertEqual([False, True, True], [r[0] for r in results])
|
||||
self.client.indices.refresh(index="i")
|
||||
res = self.client.search(index="i")
|
||||
self.assertEqual({"value": 2, "relation": "eq"}, res["hits"]["total"])
|
||||
self.assertEqual(4, failing_client._called)
|
||||
|
||||
def test_transport_error_is_raised_with_max_retries(self):
|
||||
failing_client = FailingBulkClient(
|
||||
self.client,
|
||||
fail_at=(1, 2, 3, 4),
|
||||
fail_with=TransportError(429, "Rejected!", {}),
|
||||
)
|
||||
|
||||
def streaming_bulk():
|
||||
results = list(
|
||||
helpers.streaming_bulk(
|
||||
failing_client,
|
||||
[{"a": 42}, {"a": 39}],
|
||||
raise_on_exception=True,
|
||||
max_retries=3,
|
||||
initial_backoff=0,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
self.assertRaises(TransportError, streaming_bulk)
|
||||
self.assertEqual(4, failing_client._called)
|
||||
|
||||
|
||||
class TestBulk(OpenSearchTestCase):
|
||||
def test_bulk_works_with_single_item(self):
|
||||
docs = [{"answer": 42, "_id": 1}]
|
||||
success, failed = helpers.bulk(
|
||||
self.client, docs, index="test-index", refresh=True
|
||||
)
|
||||
|
||||
self.assertEqual(1, success)
|
||||
self.assertFalse(failed)
|
||||
self.assertEqual(1, self.client.count(index="test-index")["count"])
|
||||
self.assertEqual(
|
||||
{"answer": 42}, self.client.get(index="test-index", id=1)["_source"]
|
||||
)
|
||||
|
||||
def test_all_documents_get_inserted(self):
|
||||
docs = [{"answer": x, "_id": x} for x in range(100)]
|
||||
success, failed = helpers.bulk(
|
||||
self.client, docs, index="test-index", refresh=True
|
||||
)
|
||||
|
||||
self.assertEqual(100, success)
|
||||
self.assertFalse(failed)
|
||||
self.assertEqual(100, self.client.count(index="test-index")["count"])
|
||||
self.assertEqual(
|
||||
{"answer": 42}, self.client.get(index="test-index", id=42)["_source"]
|
||||
)
|
||||
|
||||
def test_stats_only_reports_numbers(self):
|
||||
docs = [{"answer": x} for x in range(100)]
|
||||
success, failed = helpers.bulk(
|
||||
self.client, docs, index="test-index", refresh=True, stats_only=True
|
||||
)
|
||||
|
||||
self.assertEqual(100, success)
|
||||
self.assertEqual(0, failed)
|
||||
self.assertEqual(100, self.client.count(index="test-index")["count"])
|
||||
|
||||
def test_errors_are_reported_correctly(self):
|
||||
self.client.indices.create(
|
||||
"i",
|
||||
{
|
||||
"mappings": {"properties": {"a": {"type": "integer"}}},
|
||||
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
|
||||
},
|
||||
)
|
||||
self.client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
success, failed = helpers.bulk(
|
||||
self.client,
|
||||
[{"a": 42}, {"a": "c", "_id": 42}],
|
||||
index="i",
|
||||
raise_on_error=False,
|
||||
)
|
||||
self.assertEqual(1, success)
|
||||
self.assertEqual(1, len(failed))
|
||||
error = failed[0]
|
||||
self.assertEqual("42", error["index"]["_id"])
|
||||
self.assertEqual("_doc", error["index"]["_type"])
|
||||
self.assertEqual("i", error["index"]["_index"])
|
||||
print(error["index"]["error"])
|
||||
self.assertTrue(
|
||||
"MapperParsingException" in repr(error["index"]["error"])
|
||||
or "mapper_parsing_exception" in repr(error["index"]["error"])
|
||||
)
|
||||
|
||||
def test_error_is_raised(self):
|
||||
self.client.indices.create(
|
||||
"i",
|
||||
{
|
||||
"mappings": {"properties": {"a": {"type": "integer"}}},
|
||||
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
|
||||
},
|
||||
)
|
||||
self.client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
self.assertRaises(
|
||||
helpers.BulkIndexError,
|
||||
helpers.bulk,
|
||||
self.client,
|
||||
[{"a": 42}, {"a": "c"}],
|
||||
index="i",
|
||||
)
|
||||
|
||||
def test_ignore_error_if_raised(self):
|
||||
# ignore the status code 400 in tuple
|
||||
helpers.bulk(
|
||||
self.client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=(400,)
|
||||
)
|
||||
|
||||
# ignore the status code 400 in list
|
||||
helpers.bulk(
|
||||
self.client,
|
||||
[{"a": 42}, {"a": "c"}],
|
||||
index="i",
|
||||
ignore_status=[
|
||||
400,
|
||||
],
|
||||
)
|
||||
|
||||
# ignore the status code 400
|
||||
helpers.bulk(self.client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=400)
|
||||
|
||||
# ignore only the status code in the `ignore_status` argument
|
||||
self.assertRaises(
|
||||
helpers.BulkIndexError,
|
||||
helpers.bulk,
|
||||
self.client,
|
||||
[{"a": 42}, {"a": "c"}],
|
||||
index="i",
|
||||
ignore_status=(444,),
|
||||
)
|
||||
|
||||
# ignore transport error exception
|
||||
failing_client = FailingBulkClient(self.client)
|
||||
helpers.bulk(failing_client, [{"a": 42}], index="i", ignore_status=(599,))
|
||||
|
||||
def test_errors_are_collected_properly(self):
|
||||
self.client.indices.create(
|
||||
"i",
|
||||
{
|
||||
"mappings": {"properties": {"a": {"type": "integer"}}},
|
||||
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
|
||||
},
|
||||
)
|
||||
self.client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
success, failed = helpers.bulk(
|
||||
self.client,
|
||||
[{"a": 42}, {"a": "c"}],
|
||||
index="i",
|
||||
stats_only=True,
|
||||
raise_on_error=False,
|
||||
)
|
||||
self.assertEqual(1, success)
|
||||
self.assertEqual(1, failed)
|
||||
|
||||
|
||||
class TestScan(OpenSearchTestCase):
|
||||
mock_scroll_responses = [
|
||||
{
|
||||
"_scroll_id": "dummy_id",
|
||||
"_shards": {"successful": 4, "total": 5, "skipped": 0},
|
||||
"hits": {"hits": [{"scroll_data": 42}]},
|
||||
},
|
||||
{
|
||||
"_scroll_id": "dummy_id",
|
||||
"_shards": {"successful": 4, "total": 5, "skipped": 0},
|
||||
"hits": {"hits": []},
|
||||
},
|
||||
]
|
||||
|
||||
def teardown_method(self, m):
|
||||
self.client.transport.perform_request("DELETE", "/_search/scroll/_all")
|
||||
super(TestScan, self).teardown_method(m)
|
||||
|
||||
def test_order_can_be_preserved(self):
|
||||
bulk = []
|
||||
for x in range(100):
|
||||
bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}})
|
||||
bulk.append({"answer": x, "correct": x == 42})
|
||||
self.client.bulk(bulk, refresh=True)
|
||||
|
||||
docs = list(
|
||||
helpers.scan(
|
||||
self.client,
|
||||
index="test_index",
|
||||
query={"sort": "answer"},
|
||||
preserve_order=True,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(100, len(docs))
|
||||
self.assertEqual(list(map(str, range(100))), list(d["_id"] for d in docs))
|
||||
self.assertEqual(list(range(100)), list(d["_source"]["answer"] for d in docs))
|
||||
|
||||
def test_all_documents_are_read(self):
|
||||
bulk = []
|
||||
for x in range(100):
|
||||
bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}})
|
||||
bulk.append({"answer": x, "correct": x == 42})
|
||||
self.client.bulk(bulk, refresh=True)
|
||||
|
||||
docs = list(helpers.scan(self.client, index="test_index", size=2))
|
||||
|
||||
self.assertEqual(100, len(docs))
|
||||
self.assertEqual(set(map(str, range(100))), set(d["_id"] for d in docs))
|
||||
self.assertEqual(set(range(100)), set(d["_source"]["answer"] for d in docs))
|
||||
|
||||
def test_scroll_error(self):
|
||||
bulk = []
|
||||
for x in range(4):
|
||||
bulk.append({"index": {"_index": "test_index", "_type": "_doc"}})
|
||||
bulk.append({"value": x})
|
||||
self.client.bulk(bulk, refresh=True)
|
||||
|
||||
with patch.object(self.client, "scroll") as scroll_mock:
|
||||
scroll_mock.side_effect = self.mock_scroll_responses
|
||||
data = list(
|
||||
helpers.scan(
|
||||
self.client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
raise_on_error=False,
|
||||
clear_scroll=False,
|
||||
)
|
||||
)
|
||||
self.assertEqual(len(data), 3)
|
||||
self.assertEqual(data[-1], {"scroll_data": 42})
|
||||
|
||||
scroll_mock.side_effect = self.mock_scroll_responses
|
||||
with self.assertRaises(ScanError):
|
||||
data = list(
|
||||
helpers.scan(
|
||||
self.client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
raise_on_error=True,
|
||||
clear_scroll=False,
|
||||
)
|
||||
)
|
||||
self.assertEqual(len(data), 3)
|
||||
self.assertEqual(data[-1], {"scroll_data": 42})
|
||||
|
||||
def test_initial_search_error(self):
|
||||
with patch.object(self, "client") as client_mock:
|
||||
client_mock.search.return_value = {
|
||||
"_scroll_id": "dummy_id",
|
||||
"_shards": {"successful": 4, "total": 5, "skipped": 0},
|
||||
"hits": {"hits": [{"search_data": 1}]},
|
||||
}
|
||||
client_mock.scroll.side_effect = self.mock_scroll_responses
|
||||
|
||||
data = list(
|
||||
helpers.scan(
|
||||
self.client, index="test_index", size=2, raise_on_error=False
|
||||
)
|
||||
)
|
||||
self.assertEqual(data, [{"search_data": 1}, {"scroll_data": 42}])
|
||||
|
||||
client_mock.scroll.side_effect = self.mock_scroll_responses
|
||||
with self.assertRaises(ScanError):
|
||||
data = list(
|
||||
helpers.scan(
|
||||
self.client, index="test_index", size=2, raise_on_error=True
|
||||
)
|
||||
)
|
||||
self.assertEqual(data, [{"search_data": 1}])
|
||||
client_mock.scroll.assert_not_called()
|
||||
|
||||
def test_no_scroll_id_fast_route(self):
|
||||
with patch.object(self, "client") as client_mock:
|
||||
client_mock.search.return_value = {"no": "_scroll_id"}
|
||||
data = list(helpers.scan(self.client, index="test_index"))
|
||||
|
||||
self.assertEqual(data, [])
|
||||
client_mock.scroll.assert_not_called()
|
||||
client_mock.clear_scroll.assert_not_called()
|
||||
|
||||
def test_scan_auth_kwargs_forwarded(self):
|
||||
for key, val in {
|
||||
"api_key": ("name", "value"),
|
||||
"http_auth": ("username", "password"),
|
||||
"headers": {"custom": "header"},
|
||||
}.items():
|
||||
with patch.object(self, "client") as client_mock:
|
||||
client_mock.search.return_value = {
|
||||
"_scroll_id": "scroll_id",
|
||||
"_shards": {"successful": 5, "total": 5, "skipped": 0},
|
||||
"hits": {"hits": [{"search_data": 1}]},
|
||||
}
|
||||
client_mock.scroll.return_value = {
|
||||
"_scroll_id": "scroll_id",
|
||||
"_shards": {"successful": 5, "total": 5, "skipped": 0},
|
||||
"hits": {"hits": []},
|
||||
}
|
||||
client_mock.clear_scroll.return_value = {}
|
||||
|
||||
data = list(helpers.scan(self.client, index="test_index", **{key: val}))
|
||||
|
||||
self.assertEqual(data, [{"search_data": 1}])
|
||||
|
||||
# Assert that 'search', 'scroll' and 'clear_scroll' all
|
||||
# received the extra kwarg related to authentication.
|
||||
for api_mock in (
|
||||
client_mock.search,
|
||||
client_mock.scroll,
|
||||
client_mock.clear_scroll,
|
||||
):
|
||||
self.assertEqual(api_mock.call_args[1][key], val)
|
||||
|
||||
def test_scan_auth_kwargs_favor_scroll_kwargs_option(self):
|
||||
with patch.object(self, "client") as client_mock:
|
||||
client_mock.search.return_value = {
|
||||
"_scroll_id": "scroll_id",
|
||||
"_shards": {"successful": 5, "total": 5, "skipped": 0},
|
||||
"hits": {"hits": [{"search_data": 1}]},
|
||||
}
|
||||
client_mock.scroll.return_value = {
|
||||
"_scroll_id": "scroll_id",
|
||||
"_shards": {"successful": 5, "total": 5, "skipped": 0},
|
||||
"hits": {"hits": []},
|
||||
}
|
||||
client_mock.clear_scroll.return_value = {}
|
||||
|
||||
data = list(
|
||||
helpers.scan(
|
||||
self.client,
|
||||
index="test_index",
|
||||
scroll_kwargs={"headers": {"scroll": "kwargs"}, "sort": "asc"},
|
||||
headers={"not scroll": "kwargs"},
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(data, [{"search_data": 1}])
|
||||
|
||||
# Assert that we see 'scroll_kwargs' options used instead of 'kwargs'
|
||||
self.assertEqual(
|
||||
client_mock.scroll.call_args[1]["headers"], {"scroll": "kwargs"}
|
||||
)
|
||||
self.assertEqual(client_mock.scroll.call_args[1]["sort"], "asc")
|
||||
|
||||
@patch("opensearchpy.helpers.actions.logger")
|
||||
def test_logger(self, logger_mock):
|
||||
bulk = []
|
||||
for x in range(4):
|
||||
bulk.append({"index": {"_index": "test_index", "_type": "_doc"}})
|
||||
bulk.append({"value": x})
|
||||
self.client.bulk(bulk, refresh=True)
|
||||
|
||||
with patch.object(self.client, "scroll") as scroll_mock:
|
||||
scroll_mock.side_effect = self.mock_scroll_responses
|
||||
list(
|
||||
helpers.scan(
|
||||
self.client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
raise_on_error=False,
|
||||
clear_scroll=False,
|
||||
)
|
||||
)
|
||||
logger_mock.warning.assert_called()
|
||||
|
||||
scroll_mock.side_effect = self.mock_scroll_responses
|
||||
try:
|
||||
list(
|
||||
helpers.scan(
|
||||
self.client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
raise_on_error=True,
|
||||
clear_scroll=False,
|
||||
)
|
||||
)
|
||||
except ScanError:
|
||||
pass
|
||||
logger_mock.warning.assert_called()
|
||||
|
||||
def test_clear_scroll(self):
|
||||
bulk = []
|
||||
for x in range(4):
|
||||
bulk.append({"index": {"_index": "test_index", "_type": "_doc"}})
|
||||
bulk.append({"value": x})
|
||||
self.client.bulk(bulk, refresh=True)
|
||||
|
||||
with patch.object(
|
||||
self.client, "clear_scroll", wraps=self.client.clear_scroll
|
||||
) as spy:
|
||||
list(helpers.scan(self.client, index="test_index", size=2))
|
||||
spy.assert_called_once()
|
||||
|
||||
spy.reset_mock()
|
||||
list(
|
||||
helpers.scan(self.client, index="test_index", size=2, clear_scroll=True)
|
||||
)
|
||||
spy.assert_called_once()
|
||||
|
||||
spy.reset_mock()
|
||||
list(
|
||||
helpers.scan(
|
||||
self.client, index="test_index", size=2, clear_scroll=False
|
||||
)
|
||||
)
|
||||
spy.assert_not_called()
|
||||
|
||||
def test_shards_no_skipped_field(self):
|
||||
with patch.object(self, "client") as client_mock:
|
||||
client_mock.search.return_value = {
|
||||
"_scroll_id": "dummy_id",
|
||||
"_shards": {"successful": 5, "total": 5},
|
||||
"hits": {"hits": [{"search_data": 1}]},
|
||||
}
|
||||
client_mock.scroll.side_effect = [
|
||||
{
|
||||
"_scroll_id": "dummy_id",
|
||||
"_shards": {"successful": 5, "total": 5},
|
||||
"hits": {"hits": [{"scroll_data": 42}]},
|
||||
},
|
||||
{
|
||||
"_scroll_id": "dummy_id",
|
||||
"_shards": {"successful": 5, "total": 5},
|
||||
"hits": {"hits": []},
|
||||
},
|
||||
]
|
||||
|
||||
data = list(
|
||||
helpers.scan(
|
||||
self.client, index="test_index", size=2, raise_on_error=True
|
||||
)
|
||||
)
|
||||
self.assertEqual(data, [{"search_data": 1}, {"scroll_data": 42}])
|
||||
|
||||
|
||||
class TestReindex(OpenSearchTestCase):
|
||||
def setup_method(self, _):
|
||||
bulk = []
|
||||
for x in range(100):
|
||||
bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}})
|
||||
bulk.append(
|
||||
{
|
||||
"answer": x,
|
||||
"correct": x == 42,
|
||||
"type": "answers" if x % 2 == 0 else "questions",
|
||||
}
|
||||
)
|
||||
self.client.bulk(bulk, refresh=True)
|
||||
|
||||
def test_reindex_passes_kwargs_to_scan_and_bulk(self):
|
||||
helpers.reindex(
|
||||
self.client,
|
||||
"test_index",
|
||||
"prod_index",
|
||||
scan_kwargs={"q": "type:answers"},
|
||||
bulk_kwargs={"refresh": True},
|
||||
)
|
||||
|
||||
self.assertTrue(self.client.indices.exists("prod_index"))
|
||||
self.assertEqual(
|
||||
50, self.client.count(index="prod_index", q="type:answers")["count"]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{"answer": 42, "correct": True, "type": "answers"},
|
||||
self.client.get(index="prod_index", id=42)["_source"],
|
||||
)
|
||||
|
||||
def test_reindex_accepts_a_query(self):
|
||||
helpers.reindex(
|
||||
self.client,
|
||||
"test_index",
|
||||
"prod_index",
|
||||
query={"query": {"bool": {"filter": {"term": {"type": "answers"}}}}},
|
||||
)
|
||||
self.client.indices.refresh()
|
||||
|
||||
self.assertTrue(self.client.indices.exists("prod_index"))
|
||||
self.assertEqual(
|
||||
50, self.client.count(index="prod_index", q="type:answers")["count"]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{"answer": 42, "correct": True, "type": "answers"},
|
||||
self.client.get(index="prod_index", id=42)["_source"],
|
||||
)
|
||||
|
||||
def test_all_documents_get_moved(self):
|
||||
helpers.reindex(self.client, "test_index", "prod_index")
|
||||
self.client.indices.refresh()
|
||||
|
||||
self.assertTrue(self.client.indices.exists("prod_index"))
|
||||
self.assertEqual(
|
||||
50, self.client.count(index="prod_index", q="type:questions")["count"]
|
||||
)
|
||||
self.assertEqual(
|
||||
50, self.client.count(index="prod_index", q="type:answers")["count"]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{"answer": 42, "correct": True, "type": "answers"},
|
||||
self.client.get(index="prod_index", id=42)["_source"],
|
||||
)
|
||||
|
||||
|
||||
class TestParentChildReindex(OpenSearchTestCase):
|
||||
def setup_method(self, _):
|
||||
body = {
|
||||
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"question_answer": {
|
||||
"type": "join",
|
||||
"relations": {"question": "answer"},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
self.client.indices.create(index="test-index", body=body)
|
||||
self.client.indices.create(index="real-index", body=body)
|
||||
|
||||
self.client.index(
|
||||
index="test-index", id=42, body={"question_answer": "question"}
|
||||
)
|
||||
self.client.index(
|
||||
index="test-index",
|
||||
id=47,
|
||||
routing=42,
|
||||
body={"some": "data", "question_answer": {"name": "answer", "parent": 42}},
|
||||
)
|
||||
self.client.indices.refresh(index="test-index")
|
||||
|
||||
def test_children_are_reindexed_correctly(self):
|
||||
helpers.reindex(self.client, "test-index", "real-index")
|
||||
|
||||
q = self.client.get(index="real-index", id=42)
|
||||
self.assertEqual(
|
||||
{
|
||||
"_id": "42",
|
||||
"_index": "real-index",
|
||||
"_primary_term": 1,
|
||||
"_seq_no": 0,
|
||||
"_source": {"question_answer": "question"},
|
||||
"_type": "_doc",
|
||||
"_version": 1,
|
||||
"found": True,
|
||||
},
|
||||
q,
|
||||
)
|
||||
q = self.client.get(index="test-index", id=47, routing=42)
|
||||
self.assertEqual(
|
||||
{
|
||||
"_routing": "42",
|
||||
"_id": "47",
|
||||
"_index": "test-index",
|
||||
"_primary_term": 1,
|
||||
"_seq_no": 1,
|
||||
"_source": {
|
||||
"some": "data",
|
||||
"question_answer": {"name": "answer", "parent": 42},
|
||||
},
|
||||
"_type": "_doc",
|
||||
"_version": 1,
|
||||
"found": True,
|
||||
},
|
||||
q,
|
||||
)
|
||||
@@ -0,0 +1,566 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
#
|
||||
# Licensed to Elasticsearch B.V. under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch B.V. licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Dynamically generated set of TestCases based on set of yaml files describing
|
||||
some integration tests. These files are shared among all official OpenSearch
|
||||
clients.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
import urllib3
|
||||
import yaml
|
||||
|
||||
from opensearchpy import OpenSearchWarning, TransportError
|
||||
from opensearchpy.client.utils import _base64_auth_header
|
||||
from opensearchpy.compat import string_types
|
||||
from opensearchpy.helpers.test import _get_version
|
||||
|
||||
from . import get_client
|
||||
|
||||
# some params had to be changed in python, keep track of them so we can rename
|
||||
# those in the tests accordingly
|
||||
PARAMS_RENAMES = {"type": "doc_type", "from": "from_"}
|
||||
|
||||
# mapping from catch values to http status codes
|
||||
CATCH_CODES = {"missing": 404, "conflict": 409, "unauthorized": 401}
|
||||
|
||||
# test features we have implemented
|
||||
IMPLEMENTED_FEATURES = {
|
||||
"gtelte",
|
||||
"stash_in_path",
|
||||
"headers",
|
||||
"catch_unauthorized",
|
||||
"default_shards",
|
||||
"warnings",
|
||||
"allowed_warnings",
|
||||
"contains",
|
||||
"arbitrary_key",
|
||||
"transform_and_set",
|
||||
}
|
||||
|
||||
# broken YAML tests on some releases
|
||||
SKIP_TESTS = {
|
||||
# Warning about date_histogram.interval deprecation is raised randomly
|
||||
"search/aggregation/250_moving_fn[1]",
|
||||
# body: null
|
||||
"indices/simulate_index_template/10_basic[2]",
|
||||
# No ML node with sufficient capacity / random ML failing
|
||||
"ml/start_stop_datafeed",
|
||||
"ml/post_data",
|
||||
"ml/jobs_crud",
|
||||
"ml/datafeeds_crud",
|
||||
"ml/set_upgrade_mode",
|
||||
"ml/reset_job[2]",
|
||||
"ml/jobs_get_stats",
|
||||
"ml/get_datafeed_stats",
|
||||
"ml/get_trained_model_stats",
|
||||
"ml/delete_job_force",
|
||||
"ml/jobs_get_result_overall_buckets",
|
||||
"ml/bucket_correlation_agg[0]",
|
||||
"ml/job_groups",
|
||||
"transform/transforms_stats_continuous[0]",
|
||||
# Fails bad request instead of 404?
|
||||
"ml/inference_crud",
|
||||
# Our TLS certs are custom
|
||||
"ssl/10_basic[0]",
|
||||
# Our user is custom
|
||||
"users/10_basic[3]",
|
||||
# Shards/snapshots aren't right?
|
||||
"searchable_snapshots/10_usage[1]",
|
||||
# flaky data streams?
|
||||
"data_stream/10_basic[1]",
|
||||
"data_stream/80_resolve_index_data_streams[1]",
|
||||
# bad formatting?
|
||||
"cat/allocation/10_basic",
|
||||
# service account number not right?
|
||||
"service_accounts/10_basic[1]",
|
||||
# doesn't use 'contains' properly?
|
||||
"privileges/40_get_user_privs[0]",
|
||||
"privileges/40_get_user_privs[1]",
|
||||
# bad use of 'is_false'?
|
||||
"indices/get_alias/10_basic[22]",
|
||||
# unique usage of 'set'
|
||||
"indices/stats/50_disk_usage[0]",
|
||||
"indices/stats/60_field_usage[0]",
|
||||
}
|
||||
|
||||
|
||||
OPENSEARCH_VERSION = None
|
||||
RUN_ASYNC_REST_API_TESTS = (
|
||||
sys.version_info >= (3, 6)
|
||||
and os.environ.get("PYTHON_CONNECTION_CLASS") == "RequestsHttpConnection"
|
||||
)
|
||||
|
||||
FALSEY_VALUES = ("", None, False, 0, 0.0)
|
||||
|
||||
|
||||
class YamlRunner:
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
self.last_response = None
|
||||
|
||||
self._run_code = None
|
||||
self._setup_code = None
|
||||
self._teardown_code = None
|
||||
self._state = {}
|
||||
|
||||
def use_spec(self, test_spec):
|
||||
self._setup_code = test_spec.pop("setup", None)
|
||||
self._run_code = test_spec.pop("run", None)
|
||||
self._teardown_code = test_spec.pop("teardown", None)
|
||||
|
||||
def setup(self):
|
||||
# Pull skips from individual tests to not do unnecessary setup.
|
||||
skip_code = []
|
||||
for action in self._run_code:
|
||||
assert len(action) == 1
|
||||
action_type, _ = list(action.items())[0]
|
||||
if action_type == "skip":
|
||||
skip_code.append(action)
|
||||
else:
|
||||
break
|
||||
|
||||
if self._setup_code or skip_code:
|
||||
self.section("setup")
|
||||
if skip_code:
|
||||
self.run_code(skip_code)
|
||||
if self._setup_code:
|
||||
self.run_code(self._setup_code)
|
||||
|
||||
def teardown(self):
|
||||
if self._teardown_code:
|
||||
self.section("teardown")
|
||||
self.run_code(self._teardown_code)
|
||||
|
||||
def opensearch_version(self):
|
||||
global OPENSEARCH_VERSION
|
||||
if OPENSEARCH_VERSION is None:
|
||||
version_string = (self.client.info())["version"]["number"]
|
||||
if "." not in version_string:
|
||||
return ()
|
||||
version = version_string.strip().split(".")
|
||||
OPENSEARCH_VERSION = tuple(int(v) if v.isdigit() else 999 for v in version)
|
||||
return OPENSEARCH_VERSION
|
||||
|
||||
def section(self, name):
|
||||
print(("=" * 10) + " " + name + " " + ("=" * 10))
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
self.setup()
|
||||
self.section("test")
|
||||
self.run_code(self._run_code)
|
||||
finally:
|
||||
try:
|
||||
self.teardown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def run_code(self, test):
|
||||
"""Execute an instruction based on it's type."""
|
||||
for action in test:
|
||||
assert len(action) == 1
|
||||
action_type, action = list(action.items())[0]
|
||||
print(action_type, action)
|
||||
|
||||
if hasattr(self, "run_" + action_type):
|
||||
getattr(self, "run_" + action_type)(action)
|
||||
else:
|
||||
raise RuntimeError("Invalid action type %r" % (action_type,))
|
||||
|
||||
def run_do(self, action):
|
||||
api = self.client
|
||||
headers = action.pop("headers", None)
|
||||
catch = action.pop("catch", None)
|
||||
warn = action.pop("warnings", ())
|
||||
allowed_warnings = action.pop("allowed_warnings", ())
|
||||
assert len(action) == 1
|
||||
|
||||
# Remove the x_pack_rest_user authentication
|
||||
# if it's given via headers. We're already authenticated
|
||||
# via the 'elastic' user.
|
||||
if (
|
||||
headers
|
||||
and headers.get("Authorization", None)
|
||||
== "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA=="
|
||||
):
|
||||
headers.pop("Authorization")
|
||||
|
||||
method, args = list(action.items())[0]
|
||||
args["headers"] = headers
|
||||
|
||||
# locate api endpoint
|
||||
for m in method.split("."):
|
||||
assert hasattr(api, m)
|
||||
api = getattr(api, m)
|
||||
|
||||
# some parameters had to be renamed to not clash with python builtins,
|
||||
# compensate
|
||||
for k in PARAMS_RENAMES:
|
||||
if k in args:
|
||||
args[PARAMS_RENAMES[k]] = args.pop(k)
|
||||
|
||||
# resolve vars
|
||||
for k in args:
|
||||
args[k] = self._resolve(args[k])
|
||||
|
||||
warnings.simplefilter("always", category=OpenSearchWarning)
|
||||
with warnings.catch_warnings(record=True) as caught_warnings:
|
||||
try:
|
||||
self.last_response = api(**args)
|
||||
except Exception as e:
|
||||
if not catch:
|
||||
raise
|
||||
self.run_catch(catch, e)
|
||||
else:
|
||||
if catch:
|
||||
raise AssertionError(
|
||||
"Failed to catch %r in %r." % (catch, self.last_response)
|
||||
)
|
||||
|
||||
# Filter out warnings raised by other components.
|
||||
caught_warnings = [
|
||||
str(w.message)
|
||||
for w in caught_warnings
|
||||
if w.category == OpenSearchWarning
|
||||
and str(w.message) not in allowed_warnings
|
||||
]
|
||||
|
||||
# This warning can show up in many places but isn't accounted for
|
||||
# in tests, so we remove it to make sure things pass.
|
||||
include_type_name_warning = (
|
||||
"[types removal] Using include_type_name in create index requests is deprecated. "
|
||||
"The parameter will be removed in the next major version."
|
||||
)
|
||||
if (
|
||||
include_type_name_warning in caught_warnings
|
||||
and include_type_name_warning not in warn
|
||||
):
|
||||
caught_warnings.remove(include_type_name_warning)
|
||||
|
||||
# Sorting removes the issue with order raised. We only care about
|
||||
# if all warnings are raised in the single API call.
|
||||
if warn and sorted(warn) != sorted(caught_warnings):
|
||||
raise AssertionError(
|
||||
"Expected warnings not equal to actual warnings: expected=%r actual=%r"
|
||||
% (warn, caught_warnings)
|
||||
)
|
||||
|
||||
def run_catch(self, catch, exception):
|
||||
if catch == "param":
|
||||
assert isinstance(exception, TypeError)
|
||||
return
|
||||
|
||||
assert isinstance(exception, TransportError)
|
||||
if catch in CATCH_CODES:
|
||||
assert CATCH_CODES[catch] == exception.status_code
|
||||
elif catch[0] == "/" and catch[-1] == "/":
|
||||
assert (
|
||||
re.search(catch[1:-1], exception.error + " " + repr(exception.info)),
|
||||
"%s not in %r" % (catch, exception.info),
|
||||
) is not None
|
||||
self.last_response = exception.info
|
||||
|
||||
def run_skip(self, skip):
|
||||
global IMPLEMENTED_FEATURES
|
||||
|
||||
if "features" in skip:
|
||||
features = skip["features"]
|
||||
if not isinstance(features, (tuple, list)):
|
||||
features = [features]
|
||||
for feature in features:
|
||||
if feature in IMPLEMENTED_FEATURES:
|
||||
continue
|
||||
pytest.skip("feature '%s' is not supported" % feature)
|
||||
|
||||
if "version" in skip:
|
||||
version, reason = skip["version"], skip["reason"]
|
||||
if version == "all":
|
||||
pytest.skip(reason)
|
||||
min_version, max_version = version.split("-")
|
||||
min_version = _get_version(min_version) or (0,)
|
||||
max_version = _get_version(max_version) or (999,)
|
||||
if min_version <= (self.opensearch_version()) <= max_version:
|
||||
pytest.skip(reason)
|
||||
|
||||
def run_gt(self, action):
|
||||
for key, value in action.items():
|
||||
value = self._resolve(value)
|
||||
assert self._lookup(key) > value
|
||||
|
||||
def run_gte(self, action):
|
||||
for key, value in action.items():
|
||||
value = self._resolve(value)
|
||||
assert self._lookup(key) >= value
|
||||
|
||||
def run_lt(self, action):
|
||||
for key, value in action.items():
|
||||
value = self._resolve(value)
|
||||
assert self._lookup(key) < value
|
||||
|
||||
def run_lte(self, action):
|
||||
for key, value in action.items():
|
||||
value = self._resolve(value)
|
||||
assert self._lookup(key) <= value
|
||||
|
||||
def run_set(self, action):
|
||||
for key, value in action.items():
|
||||
value = self._resolve(value)
|
||||
self._state[value] = self._lookup(key)
|
||||
|
||||
def run_is_false(self, action):
|
||||
try:
|
||||
value = self._lookup(action)
|
||||
except AssertionError:
|
||||
pass
|
||||
else:
|
||||
assert value in FALSEY_VALUES
|
||||
|
||||
def run_is_true(self, action):
|
||||
value = self._lookup(action)
|
||||
assert value not in FALSEY_VALUES
|
||||
|
||||
def run_length(self, action):
|
||||
for path, expected in action.items():
|
||||
value = self._lookup(path)
|
||||
expected = self._resolve(expected)
|
||||
assert expected == len(value)
|
||||
|
||||
def run_match(self, action):
|
||||
for path, expected in action.items():
|
||||
value = self._lookup(path)
|
||||
expected = self._resolve(expected)
|
||||
|
||||
if (
|
||||
isinstance(expected, string_types)
|
||||
and expected.startswith("/")
|
||||
and expected.endswith("/")
|
||||
):
|
||||
expected = re.compile(expected[1:-1], re.VERBOSE | re.MULTILINE)
|
||||
assert expected.search(value), "%r does not match %r" % (
|
||||
value,
|
||||
expected,
|
||||
)
|
||||
else:
|
||||
self._assert_match_equals(value, expected)
|
||||
|
||||
def run_contains(self, action):
|
||||
for path, expected in action.items():
|
||||
value = self._lookup(path) # list[dict[str,str]] is returned
|
||||
expected = self._resolve(expected) # dict[str, str]
|
||||
|
||||
if expected not in value:
|
||||
raise AssertionError("%s is not contained by %s" % (expected, value))
|
||||
|
||||
def run_transform_and_set(self, action):
|
||||
for key, value in action.items():
|
||||
# Convert #base64EncodeCredentials(id,api_key) to ["id", "api_key"]
|
||||
if "#base64EncodeCredentials" in value:
|
||||
value = value.replace("#base64EncodeCredentials", "")
|
||||
value = value.replace("(", "").replace(")", "").split(",")
|
||||
self._state[key] = _base64_auth_header(
|
||||
(self._lookup(value[0]), self._lookup(value[1]))
|
||||
)
|
||||
|
||||
def _resolve(self, value):
|
||||
# resolve variables
|
||||
if isinstance(value, string_types) and "$" in value:
|
||||
for k, v in self._state.items():
|
||||
for key_replace in ("${" + k + "}", "$" + k):
|
||||
if value == key_replace:
|
||||
value = v
|
||||
break
|
||||
# We only do the in-string replacement if using ${...}
|
||||
elif (
|
||||
key_replace.startswith("${")
|
||||
and isinstance(value, string_types)
|
||||
and key_replace in value
|
||||
):
|
||||
value = value.replace(key_replace, v)
|
||||
break
|
||||
|
||||
if isinstance(value, string_types):
|
||||
value = value.strip()
|
||||
elif isinstance(value, dict):
|
||||
value = dict((k, self._resolve(v)) for (k, v) in value.items())
|
||||
elif isinstance(value, list):
|
||||
value = list(map(self._resolve, value))
|
||||
return value
|
||||
|
||||
def _lookup(self, path):
|
||||
# fetch the possibly nested value from last_response
|
||||
value = self.last_response
|
||||
if path == "$body":
|
||||
return value
|
||||
path = path.replace(r"\.", "\1")
|
||||
for step in path.split("."):
|
||||
if not step:
|
||||
continue
|
||||
step = step.replace("\1", ".")
|
||||
step = self._resolve(step)
|
||||
|
||||
if (
|
||||
isinstance(step, string_types)
|
||||
and step.isdigit()
|
||||
and isinstance(value, list)
|
||||
):
|
||||
step = int(step)
|
||||
assert isinstance(value, list)
|
||||
assert len(value) > step
|
||||
elif step == "_arbitrary_key_":
|
||||
return list(value.keys())[0]
|
||||
else:
|
||||
assert step in value
|
||||
value = value[step]
|
||||
return value
|
||||
|
||||
def _feature_enabled(self, name):
|
||||
return False
|
||||
|
||||
def _assert_match_equals(self, a, b):
|
||||
# Handle for large floating points with 'E'
|
||||
if isinstance(b, string_types) and isinstance(a, float) and "e" in repr(a):
|
||||
a = repr(a).replace("e+", "E")
|
||||
|
||||
assert a == b, "%r does not match %r" % (a, b)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def sync_runner(sync_client):
|
||||
return YamlRunner(sync_client)
|
||||
|
||||
|
||||
YAML_TEST_SPECS = []
|
||||
|
||||
# Try loading the REST API test specs from the Elastic Artifacts API
|
||||
try:
|
||||
# Construct the HTTP and OpenSearch client
|
||||
http = urllib3.PoolManager(retries=10)
|
||||
client = get_client()
|
||||
|
||||
# Make a request to OpenSearch for the build hash, we'll be looking for
|
||||
# an artifact with this same hash to download test specs for.
|
||||
client_info = client.info()
|
||||
version_number = client_info["version"]["number"]
|
||||
build_hash = client_info["version"]["build_hash"]
|
||||
|
||||
# Now talk to the artifacts API with the 'STACK_VERSION' environment variable
|
||||
resp = http.request(
|
||||
"GET",
|
||||
"https://artifacts-api.elastic.co/v1/versions/%s" % (version_number,),
|
||||
)
|
||||
resp = json.loads(resp.data.decode("utf-8"))
|
||||
|
||||
# Look through every build and see if one matches the commit hash
|
||||
# we're looking for. If not it's okay, we'll just use the latest and
|
||||
# hope for the best!
|
||||
builds = resp["version"]["builds"]
|
||||
for build in builds:
|
||||
if build["projects"]["opensearch"]["commit_hash"] == build_hash:
|
||||
break
|
||||
else:
|
||||
build = builds[0] # Use the latest
|
||||
|
||||
# Now we're looking for the 'rest-api-spec-<VERSION>-sources.jar' file
|
||||
# to download and extract in-memory.
|
||||
packages = build["projects"]["opensearch"]["packages"]
|
||||
for package in packages:
|
||||
if re.match(r"rest-resources-zip-.*\.zip", package):
|
||||
package_url = packages[package]["url"]
|
||||
break
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Could not find the package 'rest-resources-zip-*.zip' in build %r" % build
|
||||
)
|
||||
|
||||
# Download the zip and start reading YAML from the files in memory
|
||||
package_zip = zipfile.ZipFile(io.BytesIO(http.request("GET", package_url).data))
|
||||
for yaml_file in package_zip.namelist():
|
||||
if not re.match(r"^rest-api-spec/test/.*\.ya?ml$", yaml_file):
|
||||
continue
|
||||
yaml_tests = list(yaml.safe_load_all(package_zip.read(yaml_file)))
|
||||
|
||||
# Each file may have a "test" named 'setup' or 'teardown',
|
||||
# these sets of steps should be run at the beginning and end
|
||||
# of every other test within the file so we do one pass to capture those.
|
||||
setup_steps = teardown_steps = None
|
||||
test_numbers_and_steps = []
|
||||
test_number = 0
|
||||
|
||||
for yaml_test in yaml_tests:
|
||||
test_name, test_step = yaml_test.popitem()
|
||||
if test_name == "setup":
|
||||
setup_steps = test_step
|
||||
elif test_name == "teardown":
|
||||
teardown_steps = test_step
|
||||
else:
|
||||
test_numbers_and_steps.append((test_number, test_step))
|
||||
test_number += 1
|
||||
|
||||
# Now we combine setup, teardown, and test_steps into
|
||||
# a set of pytest.param() instances
|
||||
for test_number, test_step in test_numbers_and_steps:
|
||||
# Build the id from the name of the YAML file and
|
||||
# the number within that file. Most important step
|
||||
# is to remove most of the file path prefixes and
|
||||
# the .yml suffix.
|
||||
pytest_test_name = yaml_file.rpartition(".")[0].replace(".", "/")
|
||||
for prefix in ("rest-api-spec/", "test/", "oss/"):
|
||||
if pytest_test_name.startswith(prefix):
|
||||
pytest_test_name = pytest_test_name[len(prefix) :]
|
||||
pytest_param_id = "%s[%d]" % (pytest_test_name, test_number)
|
||||
|
||||
pytest_param = {
|
||||
"setup": setup_steps,
|
||||
"run": test_step,
|
||||
"teardown": teardown_steps,
|
||||
}
|
||||
# Skip either 'test_name' or 'test_name[x]'
|
||||
if pytest_test_name in SKIP_TESTS or pytest_param_id in SKIP_TESTS:
|
||||
pytest_param["skip"] = True
|
||||
|
||||
YAML_TEST_SPECS.append(pytest.param(pytest_param, id=pytest_param_id))
|
||||
|
||||
except Exception as e:
|
||||
warnings.warn("Could not load REST API tests: %s" % (str(e),))
|
||||
|
||||
|
||||
if not RUN_ASYNC_REST_API_TESTS:
|
||||
|
||||
@pytest.mark.parametrize("test_spec", YAML_TEST_SPECS)
|
||||
def test_rest_api_spec(test_spec, sync_runner):
|
||||
if test_spec.get("skip", False):
|
||||
pytest.skip("Manually skipped in 'SKIP_TESTS'")
|
||||
sync_runner.use_spec(test_spec)
|
||||
sync_runner.run()
|
||||
Reference in New Issue
Block a user