Added async support for helpers that are merged from opensearch-dsl-py (#329)
Signed-off-by: saimedhi <[email protected]>
This commit is contained in:
Sai Medhini Reddy Maryada
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from pytest import fixture
|
||||
from test_data import (
|
||||
DATA,
|
||||
FLAT_DATA,
|
||||
TEST_GIT_DATA,
|
||||
create_flat_git_index,
|
||||
create_git_index,
|
||||
)
|
||||
|
||||
from opensearchpy._async.helpers.actions import async_bulk
|
||||
from opensearchpy._async.helpers.test import get_test_client
|
||||
from opensearchpy.connection.async_connections import add_connection
|
||||
from test_opensearchpy.test_server.test_helpers.test_document import (
|
||||
Comment,
|
||||
History,
|
||||
PullRequest,
|
||||
User,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop():
|
||||
loop = asyncio.get_event_loop()
|
||||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@fixture(scope="session")
|
||||
async def client():
|
||||
client = await get_test_client(verify_certs=False, http_auth=("admin", "admin"))
|
||||
await add_connection("default", client)
|
||||
return client
|
||||
|
||||
|
||||
@fixture(scope="session")
|
||||
async def opensearch_version(client):
|
||||
info = await client.info()
|
||||
print(info)
|
||||
yield tuple(
|
||||
int(x)
|
||||
for x in re.match(r"^([0-9.]+)", info["version"]["number"]).group(1).split(".")
|
||||
)
|
||||
|
||||
|
||||
@fixture
|
||||
async def write_client(client):
|
||||
yield client
|
||||
await client.indices.delete("test-*", ignore=404)
|
||||
await client.indices.delete_template("test-template", ignore=404)
|
||||
|
||||
|
||||
@fixture
|
||||
async def data_client(client):
|
||||
# create mappings
|
||||
await create_git_index(client, "git")
|
||||
await create_flat_git_index(client, "flat-git")
|
||||
# load data
|
||||
await async_bulk(client, DATA, raise_on_error=True, refresh=True)
|
||||
await async_bulk(client, FLAT_DATA, raise_on_error=True, refresh=True)
|
||||
yield client
|
||||
await client.indices.delete("git", ignore=404)
|
||||
await client.indices.delete("flat-git", ignore=404)
|
||||
|
||||
|
||||
@fixture
|
||||
def pull_request(write_client):
|
||||
PullRequest.init()
|
||||
pr = PullRequest(
|
||||
_id=42,
|
||||
comments=[
|
||||
Comment(
|
||||
content="Hello World!",
|
||||
author=User(name="honzakral"),
|
||||
created_at=datetime(2018, 1, 9, 10, 17, 3, 21184),
|
||||
history=[
|
||||
History(
|
||||
timestamp=datetime(2012, 1, 1),
|
||||
diff="-Ahoj Svete!\n+Hello World!",
|
||||
)
|
||||
],
|
||||
),
|
||||
],
|
||||
created_at=datetime(2018, 1, 9, 9, 17, 3, 21184),
|
||||
)
|
||||
pr.save(refresh=True)
|
||||
return pr
|
||||
|
||||
|
||||
@fixture
|
||||
async def setup_ubq_tests(client):
|
||||
index = "test-git"
|
||||
await create_git_index(client, index)
|
||||
await async_bulk(client, TEST_GIT_DATA, raise_on_error=True, refresh=True)
|
||||
return index
|
||||
+45
-48
@@ -25,16 +25,13 @@
|
||||
# under the License.
|
||||
|
||||
|
||||
# Licensed to Elasticsearch B.V.under one or more agreements.
|
||||
# Elasticsearch B.V.licenses this file to you under the Apache 2.0 License.
|
||||
# See the LICENSE file in the project root for more information
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from mock import MagicMock, patch
|
||||
|
||||
from opensearchpy import TransportError, helpers
|
||||
from opensearchpy import TransportError
|
||||
from opensearchpy._async.helpers import actions
|
||||
from opensearchpy.helpers import BulkIndexError, ScanError
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
@@ -67,16 +64,16 @@ class FailingBulkClient(object):
|
||||
|
||||
class TestStreamingBulk(object):
|
||||
async def test_actions_remain_unchanged(self, async_client):
|
||||
actions = [{"_id": 1}, {"_id": 2}]
|
||||
async for ok, item in helpers.async_streaming_bulk(
|
||||
async_client, actions, index="test-index"
|
||||
actions1 = [{"_id": 1}, {"_id": 2}]
|
||||
async for ok, item in actions.async_streaming_bulk(
|
||||
async_client, actions1, index="test-index"
|
||||
):
|
||||
assert ok
|
||||
assert [{"_id": 1}, {"_id": 2}] == actions
|
||||
assert [{"_id": 1}, {"_id": 2}] == actions1
|
||||
|
||||
async def test_all_documents_get_inserted(self, async_client):
|
||||
docs = [{"answer": x, "_id": x} for x in range(100)]
|
||||
async for ok, item in helpers.async_streaming_bulk(
|
||||
async for ok, item in actions.async_streaming_bulk(
|
||||
async_client, docs, index="test-index", refresh=True
|
||||
):
|
||||
assert ok
|
||||
@@ -96,7 +93,7 @@ class TestStreamingBulk(object):
|
||||
for x in range(100):
|
||||
yield {"answer": x, "_id": x}
|
||||
|
||||
async for ok, item in helpers.async_streaming_bulk(
|
||||
async for ok, item in actions.async_streaming_bulk(
|
||||
async_client, async_gen(), index="test-index", refresh=True
|
||||
):
|
||||
assert ok
|
||||
@@ -110,7 +107,7 @@ class TestStreamingBulk(object):
|
||||
index="test-index", body={"query": {"match_all": {}}}
|
||||
)
|
||||
|
||||
async for ok, item in helpers.async_streaming_bulk(
|
||||
async for ok, item in actions.async_streaming_bulk(
|
||||
async_client, sync_gen(), index="test-index", refresh=True
|
||||
):
|
||||
assert ok
|
||||
@@ -131,7 +128,7 @@ class TestStreamingBulk(object):
|
||||
await async_client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
try:
|
||||
async for ok, item in helpers.async_streaming_bulk(
|
||||
async for ok, item in actions.async_streaming_bulk(
|
||||
async_client, [{"a": "b"}, {"a": "c"}], index="i", raise_on_error=True
|
||||
):
|
||||
assert ok
|
||||
@@ -148,7 +145,7 @@ class TestStreamingBulk(object):
|
||||
{"_op_type": "delete", "_index": "i", "_id": 45},
|
||||
{"_op_type": "update", "_index": "i", "_id": 42, "doc": {"answer": 42}},
|
||||
]
|
||||
async for ok, item in helpers.async_streaming_bulk(async_client, docs):
|
||||
async for ok, item in actions.async_streaming_bulk(async_client, docs):
|
||||
assert ok
|
||||
|
||||
assert not await async_client.exists(index="i", id=45)
|
||||
@@ -165,7 +162,7 @@ class TestStreamingBulk(object):
|
||||
|
||||
results = [
|
||||
x
|
||||
async for x in helpers.async_streaming_bulk(
|
||||
async for x in actions.async_streaming_bulk(
|
||||
failing_client,
|
||||
docs,
|
||||
raise_on_exception=False,
|
||||
@@ -200,7 +197,7 @@ class TestStreamingBulk(object):
|
||||
]
|
||||
results = [
|
||||
x
|
||||
async for x in helpers.async_streaming_bulk(
|
||||
async for x in actions.async_streaming_bulk(
|
||||
failing_client,
|
||||
docs,
|
||||
raise_on_exception=False,
|
||||
@@ -231,7 +228,7 @@ class TestStreamingBulk(object):
|
||||
]
|
||||
results = [
|
||||
x
|
||||
async for x in helpers.async_streaming_bulk(
|
||||
async for x in actions.async_streaming_bulk(
|
||||
failing_client,
|
||||
docs,
|
||||
raise_on_exception=False,
|
||||
@@ -258,7 +255,7 @@ class TestStreamingBulk(object):
|
||||
async def streaming_bulk():
|
||||
results = [
|
||||
x
|
||||
async for x in helpers.async_streaming_bulk(
|
||||
async for x in actions.async_streaming_bulk(
|
||||
failing_client,
|
||||
[{"a": 42}, {"a": 39}],
|
||||
raise_on_exception=True,
|
||||
@@ -276,7 +273,7 @@ class TestStreamingBulk(object):
|
||||
class TestBulk(object):
|
||||
async def test_bulk_works_with_single_item(self, async_client):
|
||||
docs = [{"answer": 42, "_id": 1}]
|
||||
success, failed = await helpers.async_bulk(
|
||||
success, failed = await actions.async_bulk(
|
||||
async_client, docs, index="test-index", refresh=True
|
||||
)
|
||||
|
||||
@@ -289,7 +286,7 @@ class TestBulk(object):
|
||||
|
||||
async def test_all_documents_get_inserted(self, async_client):
|
||||
docs = [{"answer": x, "_id": x} for x in range(100)]
|
||||
success, failed = await helpers.async_bulk(
|
||||
success, failed = await actions.async_bulk(
|
||||
async_client, docs, index="test-index", refresh=True
|
||||
)
|
||||
|
||||
@@ -302,7 +299,7 @@ class TestBulk(object):
|
||||
|
||||
async def test_stats_only_reports_numbers(self, async_client):
|
||||
docs = [{"answer": x} for x in range(100)]
|
||||
success, failed = await helpers.async_bulk(
|
||||
success, failed = await actions.async_bulk(
|
||||
async_client, docs, index="test-index", refresh=True, stats_only=True
|
||||
)
|
||||
|
||||
@@ -320,7 +317,7 @@ class TestBulk(object):
|
||||
)
|
||||
await async_client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
success, failed = await helpers.async_bulk(
|
||||
success, failed = await actions.async_bulk(
|
||||
async_client,
|
||||
[{"a": 42}, {"a": "c", "_id": 42}],
|
||||
index="i",
|
||||
@@ -347,16 +344,16 @@ class TestBulk(object):
|
||||
await async_client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
with pytest.raises(BulkIndexError):
|
||||
await helpers.async_bulk(async_client, [{"a": 42}, {"a": "c"}], index="i")
|
||||
await actions.async_bulk(async_client, [{"a": 42}, {"a": "c"}], index="i")
|
||||
|
||||
async def test_ignore_error_if_raised(self, async_client):
|
||||
# ignore the status code 400 in tuple
|
||||
await helpers.async_bulk(
|
||||
await actions.async_bulk(
|
||||
async_client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=(400,)
|
||||
)
|
||||
|
||||
# ignore the status code 400 in list
|
||||
await helpers.async_bulk(
|
||||
await actions.async_bulk(
|
||||
async_client,
|
||||
[{"a": 42}, {"a": "c"}],
|
||||
index="i",
|
||||
@@ -366,19 +363,19 @@ class TestBulk(object):
|
||||
)
|
||||
|
||||
# ignore the status code 400
|
||||
await helpers.async_bulk(
|
||||
await actions.async_bulk(
|
||||
async_client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=400
|
||||
)
|
||||
|
||||
# ignore only the status code in the `ignore_status` argument
|
||||
with pytest.raises(BulkIndexError):
|
||||
await helpers.async_bulk(
|
||||
await actions.async_bulk(
|
||||
async_client, [{"a": 42}, {"a": "c"}], index="i", ignore_status=(444,)
|
||||
)
|
||||
|
||||
# ignore transport error exception
|
||||
failing_client = FailingBulkClient(async_client)
|
||||
await helpers.async_bulk(
|
||||
await actions.async_bulk(
|
||||
failing_client, [{"a": 42}], index="i", ignore_status=(599,)
|
||||
)
|
||||
|
||||
@@ -392,7 +389,7 @@ class TestBulk(object):
|
||||
)
|
||||
await async_client.cluster.health(wait_for_status="yellow")
|
||||
|
||||
success, failed = await helpers.async_bulk(
|
||||
success, failed = await actions.async_bulk(
|
||||
async_client,
|
||||
[{"a": 42}, {"a": "c"}],
|
||||
index="i",
|
||||
@@ -452,7 +449,7 @@ class TestScan(object):
|
||||
|
||||
docs = [
|
||||
doc
|
||||
async for doc in helpers.async_scan(
|
||||
async for doc in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
query={"sort": "answer"},
|
||||
@@ -473,7 +470,7 @@ class TestScan(object):
|
||||
|
||||
docs = [
|
||||
x
|
||||
async for x in helpers.async_scan(async_client, index="test_index", size=2)
|
||||
async for x in actions.async_scan(async_client, index="test_index", size=2)
|
||||
]
|
||||
|
||||
assert 100 == len(docs)
|
||||
@@ -490,7 +487,7 @@ class TestScan(object):
|
||||
with patch.object(async_client, "scroll", MockScroll()):
|
||||
data = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
@@ -505,7 +502,7 @@ class TestScan(object):
|
||||
with pytest.raises(ScanError):
|
||||
data = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
@@ -532,7 +529,7 @@ class TestScan(object):
|
||||
with patch.object(async_client, "scroll", MockScroll()):
|
||||
data = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
@@ -556,7 +553,7 @@ class TestScan(object):
|
||||
with pytest.raises(ScanError):
|
||||
data = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
@@ -572,7 +569,7 @@ class TestScan(object):
|
||||
with patch.object(async_client, "clear_scroll") as clear_mock:
|
||||
data = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client, index="test_index"
|
||||
)
|
||||
]
|
||||
@@ -581,7 +578,7 @@ class TestScan(object):
|
||||
scroll_mock.assert_not_called()
|
||||
clear_mock.assert_not_called()
|
||||
|
||||
@patch("opensearchpy._async.helpers.logger")
|
||||
@patch("opensearchpy._async.helpers.actions.logger")
|
||||
async def test_logger(self, logger_mock, async_client, scan_teardown):
|
||||
bulk = []
|
||||
for x in range(4):
|
||||
@@ -592,7 +589,7 @@ class TestScan(object):
|
||||
with patch.object(async_client, "scroll", MockScroll()):
|
||||
_ = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
@@ -606,7 +603,7 @@ class TestScan(object):
|
||||
try:
|
||||
_ = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
size=2,
|
||||
@@ -635,7 +632,7 @@ class TestScan(object):
|
||||
) as spy:
|
||||
_ = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client, index="test_index", size=2
|
||||
)
|
||||
]
|
||||
@@ -644,7 +641,7 @@ class TestScan(object):
|
||||
spy.reset_mock()
|
||||
_ = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client, index="test_index", size=2, clear_scroll=True
|
||||
)
|
||||
]
|
||||
@@ -653,7 +650,7 @@ class TestScan(object):
|
||||
spy.reset_mock()
|
||||
_ = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client, index="test_index", size=2, clear_scroll=False
|
||||
)
|
||||
]
|
||||
@@ -699,7 +696,7 @@ class TestScan(object):
|
||||
) as clear_mock:
|
||||
data = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client, index="test_index", **kwargs
|
||||
)
|
||||
]
|
||||
@@ -739,7 +736,7 @@ class TestScan(object):
|
||||
):
|
||||
data = [
|
||||
x
|
||||
async for x in helpers.async_scan(
|
||||
async for x in actions.async_scan(
|
||||
async_client,
|
||||
index="test_index",
|
||||
headers={"not scroll": "kwargs"},
|
||||
@@ -779,7 +776,7 @@ class TestReindex(object):
|
||||
async def test_reindex_passes_kwargs_to_scan_and_bulk(
|
||||
self, async_client, reindex_setup
|
||||
):
|
||||
await helpers.async_reindex(
|
||||
await actions.async_reindex(
|
||||
async_client,
|
||||
"test_index",
|
||||
"prod_index",
|
||||
@@ -798,7 +795,7 @@ class TestReindex(object):
|
||||
)["_source"]
|
||||
|
||||
async def test_reindex_accepts_a_query(self, async_client, reindex_setup):
|
||||
await helpers.async_reindex(
|
||||
await actions.async_reindex(
|
||||
async_client,
|
||||
"test_index",
|
||||
"prod_index",
|
||||
@@ -817,7 +814,7 @@ class TestReindex(object):
|
||||
)["_source"]
|
||||
|
||||
async def test_all_documents_get_moved(self, async_client, reindex_setup):
|
||||
await helpers.async_reindex(async_client, "test_index", "prod_index")
|
||||
await actions.async_reindex(async_client, "test_index", "prod_index")
|
||||
await async_client.indices.refresh()
|
||||
|
||||
assert await async_client.indices.exists("prod_index")
|
||||
@@ -869,7 +866,7 @@ class TestParentChildReindex:
|
||||
async def test_children_are_reindexed_correctly(
|
||||
self, async_client, parent_reindex_setup
|
||||
):
|
||||
await helpers.async_reindex(async_client, "test-index", "real-index")
|
||||
await actions.async_reindex(async_client, "test-index", "real-index")
|
||||
assert {"question_answer": "question"} == (
|
||||
await async_client.get(index="real-index", id=42)
|
||||
)["_source"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,555 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
|
||||
from datetime import datetime
|
||||
from ipaddress import ip_address
|
||||
|
||||
import pytest
|
||||
from pytest import raises
|
||||
from pytz import timezone
|
||||
|
||||
from opensearchpy import (
|
||||
Binary,
|
||||
Boolean,
|
||||
ConflictError,
|
||||
Date,
|
||||
Double,
|
||||
InnerDoc,
|
||||
Ip,
|
||||
Keyword,
|
||||
Long,
|
||||
MetaField,
|
||||
Nested,
|
||||
NotFoundError,
|
||||
Object,
|
||||
Q,
|
||||
RankFeatures,
|
||||
Text,
|
||||
analyzer,
|
||||
)
|
||||
from opensearchpy._async.helpers.actions import aiter
|
||||
from opensearchpy._async.helpers.document import AsyncDocument
|
||||
from opensearchpy._async.helpers.mapping import AsyncMapping
|
||||
from opensearchpy.helpers.utils import AttrList
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
snowball = analyzer("my_snow", tokenizer="standard", filter=["lowercase", "snowball"])
|
||||
|
||||
|
||||
class User(InnerDoc):
|
||||
name = Text(fields={"raw": Keyword()})
|
||||
|
||||
|
||||
class Wiki(AsyncDocument):
|
||||
owner = Object(User)
|
||||
views = Long()
|
||||
ranked = RankFeatures()
|
||||
|
||||
class Index:
|
||||
name = "test-wiki"
|
||||
|
||||
|
||||
class Repository(AsyncDocument):
|
||||
owner = Object(User)
|
||||
created_at = Date()
|
||||
description = Text(analyzer=snowball)
|
||||
tags = Keyword()
|
||||
|
||||
@classmethod
|
||||
def search(cls):
|
||||
return super(Repository, cls).search().filter("term", commit_repo="repo")
|
||||
|
||||
class Index:
|
||||
name = "git"
|
||||
|
||||
|
||||
class Commit(AsyncDocument):
|
||||
committed_date = Date()
|
||||
authored_date = Date()
|
||||
description = Text(analyzer=snowball)
|
||||
|
||||
class Index:
|
||||
name = "flat-git"
|
||||
|
||||
class Meta:
|
||||
mapping = AsyncMapping()
|
||||
|
||||
|
||||
class History(InnerDoc):
|
||||
timestamp = Date()
|
||||
diff = Text()
|
||||
|
||||
|
||||
class Comment(InnerDoc):
|
||||
content = Text()
|
||||
created_at = Date()
|
||||
author = Object(User)
|
||||
history = Nested(History)
|
||||
|
||||
class Meta:
|
||||
dynamic = MetaField(False)
|
||||
|
||||
|
||||
class PullRequest(AsyncDocument):
|
||||
comments = Nested(Comment)
|
||||
created_at = Date()
|
||||
|
||||
class Index:
|
||||
name = "test-prs"
|
||||
|
||||
|
||||
class SerializationDoc(AsyncDocument):
|
||||
i = Long()
|
||||
b = Boolean()
|
||||
d = Double()
|
||||
bin = Binary()
|
||||
ip = Ip()
|
||||
|
||||
class Index:
|
||||
name = "test-serialization"
|
||||
|
||||
|
||||
async def test_serialization(write_client):
|
||||
await SerializationDoc.init()
|
||||
await write_client.index(
|
||||
index="test-serialization",
|
||||
id=42,
|
||||
body={
|
||||
"i": [1, 2, "3", None],
|
||||
"b": [True, False, "true", "false", None],
|
||||
"d": [0.1, "-0.1", None],
|
||||
"bin": ["SGVsbG8gV29ybGQ=", None],
|
||||
"ip": ["::1", "127.0.0.1", None],
|
||||
},
|
||||
)
|
||||
sd = await SerializationDoc.get(id=42)
|
||||
|
||||
assert sd.i == [1, 2, 3, None]
|
||||
assert sd.b == [True, False, True, False, None]
|
||||
assert sd.d == [0.1, -0.1, None]
|
||||
assert sd.bin == [b"Hello World", None]
|
||||
assert sd.ip == [ip_address("::1"), ip_address("127.0.0.1"), None]
|
||||
|
||||
assert sd.to_dict() == {
|
||||
"b": [True, False, True, False, None],
|
||||
"bin": ["SGVsbG8gV29ybGQ=", None],
|
||||
"d": [0.1, -0.1, None],
|
||||
"i": [1, 2, 3, None],
|
||||
"ip": ["::1", "127.0.0.1", None],
|
||||
}
|
||||
|
||||
|
||||
async def test_nested_inner_hits_are_wrapped_properly(pull_request):
|
||||
history_query = Q(
|
||||
"nested",
|
||||
path="comments.history",
|
||||
inner_hits={},
|
||||
query=Q("match", comments__history__diff="ahoj"),
|
||||
)
|
||||
s = PullRequest.search().query(
|
||||
"nested", inner_hits={}, path="comments", query=history_query
|
||||
)
|
||||
|
||||
response = await s.execute()
|
||||
pr = response.hits[0]
|
||||
assert isinstance(pr, PullRequest)
|
||||
assert isinstance(pr.comments[0], Comment)
|
||||
assert isinstance(pr.comments[0].history[0], History)
|
||||
|
||||
comment = pr.meta.inner_hits.comments.hits[0]
|
||||
assert isinstance(comment, Comment)
|
||||
assert comment.author.name == "honzakral"
|
||||
assert isinstance(comment.history[0], History)
|
||||
|
||||
history = comment.meta.inner_hits["comments.history"].hits[0]
|
||||
assert isinstance(history, History)
|
||||
assert history.timestamp == datetime(2012, 1, 1)
|
||||
assert "score" in history.meta
|
||||
|
||||
|
||||
async def test_nested_inner_hits_are_deserialized_properly(pull_request):
|
||||
s = PullRequest.search().query(
|
||||
"nested",
|
||||
inner_hits={},
|
||||
path="comments",
|
||||
query=Q("match", comments__content="hello"),
|
||||
)
|
||||
|
||||
response = await s.execute()
|
||||
pr = response.hits[0]
|
||||
assert isinstance(pr.created_at, datetime)
|
||||
assert isinstance(pr.comments[0], Comment)
|
||||
assert isinstance(pr.comments[0].created_at, datetime)
|
||||
|
||||
|
||||
async def test_nested_top_hits_are_wrapped_properly(pull_request):
|
||||
s = PullRequest.search()
|
||||
s.aggs.bucket("comments", "nested", path="comments").metric(
|
||||
"hits", "top_hits", size=1
|
||||
)
|
||||
|
||||
r = await s.execute()
|
||||
|
||||
print(r._d_)
|
||||
assert isinstance(r.aggregations.comments.hits.hits[0], Comment)
|
||||
|
||||
|
||||
async def test_update_object_field(write_client):
|
||||
await Wiki.init()
|
||||
w = Wiki(
|
||||
owner=User(name="Honza Kral"),
|
||||
_id="opensearch-py",
|
||||
ranked={"test1": 0.1, "topic2": 0.2},
|
||||
)
|
||||
await w.save()
|
||||
|
||||
assert "updated" == await w.update(owner=[{"name": "Honza"}, {"name": "Nick"}])
|
||||
assert w.owner[0].name == "Honza"
|
||||
assert w.owner[1].name == "Nick"
|
||||
|
||||
w = await Wiki.get(id="opensearch-py")
|
||||
assert w.owner[0].name == "Honza"
|
||||
assert w.owner[1].name == "Nick"
|
||||
|
||||
assert w.ranked == {"test1": 0.1, "topic2": 0.2}
|
||||
|
||||
|
||||
async def test_update_script(write_client):
|
||||
await Wiki.init()
|
||||
w = Wiki(owner=User(name="Honza Kral"), _id="opensearch-py", views=42)
|
||||
await w.save()
|
||||
|
||||
await w.update(script="ctx._source.views += params.inc", inc=5)
|
||||
w = await Wiki.get(id="opensearch-py")
|
||||
assert w.views == 47
|
||||
|
||||
|
||||
async def test_update_retry_on_conflict(write_client):
|
||||
await Wiki.init()
|
||||
w = Wiki(owner=User(name="Honza Kral"), _id="opensearch-py", views=42)
|
||||
await w.save()
|
||||
|
||||
w1 = await Wiki.get(id="opensearch-py")
|
||||
w2 = await Wiki.get(id="opensearch-py")
|
||||
await w1.update(
|
||||
script="ctx._source.views += params.inc", inc=5, retry_on_conflict=1
|
||||
)
|
||||
await w2.update(
|
||||
script="ctx._source.views += params.inc", inc=5, retry_on_conflict=1
|
||||
)
|
||||
|
||||
w = await Wiki.get(id="opensearch-py")
|
||||
assert w.views == 52
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retry_on_conflict", [None, 0])
|
||||
async def test_update_conflicting_version(write_client, retry_on_conflict):
|
||||
await Wiki.init()
|
||||
w = Wiki(owner=User(name="Honza Kral"), _id="opensearch-py", views=42)
|
||||
await w.save()
|
||||
|
||||
w1 = await Wiki.get(id="opensearch-py")
|
||||
w2 = await Wiki.get(id="opensearch-py")
|
||||
await w1.update(script="ctx._source.views += params.inc", inc=5)
|
||||
|
||||
with raises(ConflictError):
|
||||
await w2.update(
|
||||
script="ctx._source.views += params.inc",
|
||||
inc=5,
|
||||
retry_on_conflict=retry_on_conflict,
|
||||
)
|
||||
|
||||
|
||||
async def test_save_and_update_return_doc_meta(write_client):
|
||||
await Wiki.init()
|
||||
w = Wiki(owner=User(name="Honza Kral"), _id="opensearch-py", views=42)
|
||||
resp = await w.save(return_doc_meta=True)
|
||||
assert resp["_index"] == "test-wiki"
|
||||
assert resp["result"] == "created"
|
||||
assert resp.keys().__contains__("_id")
|
||||
assert resp.keys().__contains__("_primary_term")
|
||||
assert resp.keys().__contains__("_seq_no")
|
||||
assert resp.keys().__contains__("_shards")
|
||||
assert resp.keys().__contains__("_version")
|
||||
|
||||
resp = await w.update(
|
||||
script="ctx._source.views += params.inc", inc=5, return_doc_meta=True
|
||||
)
|
||||
assert resp["_index"] == "test-wiki"
|
||||
assert resp["result"] == "updated"
|
||||
assert resp.keys().__contains__("_id")
|
||||
assert resp.keys().__contains__("_primary_term")
|
||||
assert resp.keys().__contains__("_seq_no")
|
||||
assert resp.keys().__contains__("_shards")
|
||||
assert resp.keys().__contains__("_version")
|
||||
|
||||
|
||||
async def test_init(write_client):
|
||||
await Repository.init(index="test-git")
|
||||
|
||||
assert await write_client.indices.exists(index="test-git")
|
||||
|
||||
|
||||
async def test_get_raises_404_on_index_missing(data_client):
|
||||
with raises(NotFoundError):
|
||||
await Repository.get("opensearch-dsl-php", index="not-there")
|
||||
|
||||
|
||||
async def test_get_raises_404_on_non_existent_id(data_client):
|
||||
with raises(NotFoundError):
|
||||
await Repository.get("opensearch-dsl-php")
|
||||
|
||||
|
||||
async def test_get_returns_none_if_404_ignored(data_client):
|
||||
assert None is await Repository.get("opensearch-dsl-php", ignore=404)
|
||||
|
||||
|
||||
async def test_get_returns_none_if_404_ignored_and_index_doesnt_exist(data_client):
|
||||
assert None is await Repository.get("42", index="not-there", ignore=404)
|
||||
|
||||
|
||||
async def test_get(data_client):
|
||||
opensearch_repo = await Repository.get("opensearch-py")
|
||||
|
||||
assert isinstance(opensearch_repo, Repository)
|
||||
assert opensearch_repo.owner.name == "opensearch"
|
||||
assert datetime(2014, 3, 3) == opensearch_repo.created_at
|
||||
|
||||
|
||||
async def test_exists_return_true(data_client):
|
||||
assert await Repository.exists("opensearch-py")
|
||||
|
||||
|
||||
async def test_exists_false(data_client):
|
||||
assert not await Repository.exists("opensearch-dsl-php")
|
||||
|
||||
|
||||
async def test_get_with_tz_date(data_client):
|
||||
first_commit = await Commit.get(
|
||||
id="3ca6e1e73a071a705b4babd2f581c91a2a3e5037", routing="opensearch-py"
|
||||
)
|
||||
|
||||
tzinfo = timezone("Europe/Prague")
|
||||
assert (
|
||||
tzinfo.localize(datetime(2014, 5, 2, 13, 47, 19, 123000))
|
||||
== first_commit.authored_date
|
||||
)
|
||||
|
||||
|
||||
async def test_save_with_tz_date(data_client):
|
||||
tzinfo = timezone("Europe/Prague")
|
||||
first_commit = await Commit.get(
|
||||
id="3ca6e1e73a071a705b4babd2f581c91a2a3e5037", routing="opensearch-py"
|
||||
)
|
||||
first_commit.committed_date = tzinfo.localize(
|
||||
datetime(2014, 5, 2, 13, 47, 19, 123456)
|
||||
)
|
||||
await first_commit.save()
|
||||
|
||||
first_commit = await Commit.get(
|
||||
id="3ca6e1e73a071a705b4babd2f581c91a2a3e5037", routing="opensearch-py"
|
||||
)
|
||||
assert (
|
||||
tzinfo.localize(datetime(2014, 5, 2, 13, 47, 19, 123456))
|
||||
== first_commit.committed_date
|
||||
)
|
||||
|
||||
|
||||
COMMIT_DOCS_WITH_MISSING = [
|
||||
{"_id": "0"}, # Missing
|
||||
{"_id": "3ca6e1e73a071a705b4babd2f581c91a2a3e5037"}, # Existing
|
||||
{"_id": "f"}, # Missing
|
||||
{"_id": "eb3e543323f189fd7b698e66295427204fff5755"}, # Existing
|
||||
]
|
||||
|
||||
|
||||
async def test_mget(data_client):
|
||||
commits = await Commit.mget(COMMIT_DOCS_WITH_MISSING)
|
||||
assert commits[0] is None
|
||||
assert commits[1].meta.id == "3ca6e1e73a071a705b4babd2f581c91a2a3e5037"
|
||||
assert commits[2] is None
|
||||
assert commits[3].meta.id == "eb3e543323f189fd7b698e66295427204fff5755"
|
||||
|
||||
|
||||
async def test_mget_raises_exception_when_missing_param_is_invalid(data_client):
|
||||
with raises(ValueError):
|
||||
await Commit.mget(COMMIT_DOCS_WITH_MISSING, missing="raj")
|
||||
|
||||
|
||||
async def test_mget_raises_404_when_missing_param_is_raise(data_client):
|
||||
with raises(NotFoundError):
|
||||
await Commit.mget(COMMIT_DOCS_WITH_MISSING, missing="raise")
|
||||
|
||||
|
||||
async def test_mget_ignores_missing_docs_when_missing_param_is_skip(data_client):
|
||||
commits = await Commit.mget(COMMIT_DOCS_WITH_MISSING, missing="skip")
|
||||
assert commits[0].meta.id == "3ca6e1e73a071a705b4babd2f581c91a2a3e5037"
|
||||
assert commits[1].meta.id == "eb3e543323f189fd7b698e66295427204fff5755"
|
||||
|
||||
|
||||
async def test_update_works_from_search_response(data_client):
|
||||
opensearch_repo = (await Repository.search().execute())[0]
|
||||
|
||||
await opensearch_repo.update(owner={"other_name": "opensearchpy"})
|
||||
assert "opensearchpy" == opensearch_repo.owner.other_name
|
||||
|
||||
new_version = await Repository.get("opensearch-py")
|
||||
assert "opensearchpy" == new_version.owner.other_name
|
||||
assert "opensearch" == new_version.owner.name
|
||||
|
||||
|
||||
async def test_update(data_client):
|
||||
opensearch_repo = await Repository.get("opensearch-py")
|
||||
v = opensearch_repo.meta.version
|
||||
|
||||
old_seq_no = opensearch_repo.meta.seq_no
|
||||
await opensearch_repo.update(
|
||||
owner={"new_name": "opensearchpy"}, new_field="testing-update"
|
||||
)
|
||||
|
||||
assert "opensearchpy" == opensearch_repo.owner.new_name
|
||||
assert "testing-update" == opensearch_repo.new_field
|
||||
|
||||
# assert version has been updated
|
||||
assert opensearch_repo.meta.version == v + 1
|
||||
|
||||
new_version = await Repository.get("opensearch-py")
|
||||
assert "testing-update" == new_version.new_field
|
||||
assert "opensearchpy" == new_version.owner.new_name
|
||||
assert "opensearch" == new_version.owner.name
|
||||
assert "seq_no" in new_version.meta
|
||||
assert new_version.meta.seq_no != old_seq_no
|
||||
assert "primary_term" in new_version.meta
|
||||
|
||||
|
||||
async def test_save_updates_existing_doc(data_client):
|
||||
opensearch_repo = await Repository.get("opensearch-py")
|
||||
|
||||
opensearch_repo.new_field = "testing-save"
|
||||
old_seq_no = opensearch_repo.meta.seq_no
|
||||
assert "updated" == await opensearch_repo.save()
|
||||
|
||||
new_repo = await data_client.get(index="git", id="opensearch-py")
|
||||
assert "testing-save" == new_repo["_source"]["new_field"]
|
||||
assert new_repo["_seq_no"] != old_seq_no
|
||||
assert new_repo["_seq_no"] == opensearch_repo.meta.seq_no
|
||||
|
||||
|
||||
async def test_save_automatically_uses_seq_no_and_primary_term(data_client):
|
||||
opensearch_repo = await Repository.get("opensearch-py")
|
||||
opensearch_repo.meta.seq_no += 1
|
||||
|
||||
with raises(ConflictError):
|
||||
await opensearch_repo.save()
|
||||
|
||||
|
||||
async def test_delete_automatically_uses_seq_no_and_primary_term(data_client):
|
||||
opensearch_repo = await Repository.get("opensearch-py")
|
||||
opensearch_repo.meta.seq_no += 1
|
||||
|
||||
with raises(ConflictError):
|
||||
await opensearch_repo.delete()
|
||||
|
||||
|
||||
async def assert_doc_equals(expected, actual):
|
||||
async for f in aiter(expected):
|
||||
assert f in actual
|
||||
assert actual[f] == expected[f]
|
||||
|
||||
|
||||
async def test_can_save_to_different_index(write_client):
|
||||
test_repo = Repository(description="testing", meta={"id": 42})
|
||||
assert await test_repo.save(index="test-document")
|
||||
|
||||
await assert_doc_equals(
|
||||
{
|
||||
"found": True,
|
||||
"_index": "test-document",
|
||||
"_id": "42",
|
||||
"_source": {"description": "testing"},
|
||||
},
|
||||
await write_client.get(index="test-document", id=42),
|
||||
)
|
||||
|
||||
|
||||
async def test_save_without_skip_empty_will_include_empty_fields(write_client):
|
||||
test_repo = Repository(field_1=[], field_2=None, field_3={}, meta={"id": 42})
|
||||
assert await test_repo.save(index="test-document", skip_empty=False)
|
||||
|
||||
await assert_doc_equals(
|
||||
{
|
||||
"found": True,
|
||||
"_index": "test-document",
|
||||
"_id": "42",
|
||||
"_source": {"field_1": [], "field_2": None, "field_3": {}},
|
||||
},
|
||||
await write_client.get(index="test-document", id=42),
|
||||
)
|
||||
|
||||
|
||||
async def test_delete(write_client):
|
||||
await write_client.create(
|
||||
index="test-document",
|
||||
id="opensearch-py",
|
||||
body={
|
||||
"organization": "opensearch",
|
||||
"created_at": "2014-03-03",
|
||||
"owner": {"name": "opensearch"},
|
||||
},
|
||||
)
|
||||
|
||||
test_repo = Repository(meta={"id": "opensearch-py"})
|
||||
test_repo.meta.index = "test-document"
|
||||
await test_repo.delete()
|
||||
|
||||
assert not await write_client.exists(
|
||||
index="test-document",
|
||||
id="opensearch-py",
|
||||
)
|
||||
|
||||
|
||||
async def test_search(data_client):
|
||||
assert await Repository.search().count() == 1
|
||||
|
||||
|
||||
async def test_search_returns_proper_doc_classes(data_client):
|
||||
result = await Repository.search().execute()
|
||||
|
||||
opensearch_repo = result.hits[0]
|
||||
|
||||
assert isinstance(opensearch_repo, Repository)
|
||||
assert opensearch_repo.owner.name == "opensearch"
|
||||
|
||||
|
||||
async def test_refresh_mapping(data_client):
|
||||
class Commit(AsyncDocument):
|
||||
class Index:
|
||||
name = "git"
|
||||
|
||||
await Commit._index.load_mappings()
|
||||
|
||||
assert "stats" in Commit._index._mapping
|
||||
assert "committer" in Commit._index._mapping
|
||||
assert "description" in Commit._index._mapping
|
||||
assert "committed_date" in Commit._index._mapping
|
||||
assert isinstance(Commit._index._mapping["committed_date"], Date)
|
||||
|
||||
|
||||
async def test_highlight_in_meta(data_client):
|
||||
commit = (
|
||||
await Commit.search()
|
||||
.query("match", description="inverting")
|
||||
.highlight("description")
|
||||
.execute()
|
||||
)[0]
|
||||
|
||||
assert isinstance(commit, Commit)
|
||||
assert "description" in commit.meta.highlight
|
||||
assert isinstance(commit.meta.highlight["description"], AttrList)
|
||||
assert len(commit.meta.highlight["description"]) > 0
|
||||
@@ -0,0 +1,274 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from opensearchpy import A, Boolean, Date, Keyword
|
||||
from opensearchpy._async.helpers.document import AsyncDocument
|
||||
from opensearchpy._async.helpers.faceted_search import AsyncFacetedSearch
|
||||
from opensearchpy.helpers.faceted_search import (
|
||||
DateHistogramFacet,
|
||||
NestedFacet,
|
||||
RangeFacet,
|
||||
TermsFacet,
|
||||
)
|
||||
from test_opensearchpy.test_async.test_server.test_helpers.test_document import (
|
||||
PullRequest,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
class Repos(AsyncDocument):
|
||||
is_public = Boolean()
|
||||
created_at = Date()
|
||||
|
||||
class Index:
|
||||
name = "git"
|
||||
|
||||
|
||||
class Commit(AsyncDocument):
|
||||
files = Keyword()
|
||||
committed_date = Date()
|
||||
|
||||
class Index:
|
||||
name = "git"
|
||||
|
||||
|
||||
class MetricSearch(AsyncFacetedSearch):
|
||||
index = "git"
|
||||
doc_types = [Commit]
|
||||
|
||||
facets = {
|
||||
"files": TermsFacet(field="files", metric=A("max", field="committed_date")),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def commit_search_cls(opensearch_version):
|
||||
interval_kwargs = {"fixed_interval": "1d"}
|
||||
|
||||
class CommitSearch(AsyncFacetedSearch):
|
||||
index = "flat-git"
|
||||
fields = (
|
||||
"description",
|
||||
"files",
|
||||
)
|
||||
|
||||
facets = {
|
||||
"files": TermsFacet(field="files"),
|
||||
"frequency": DateHistogramFacet(
|
||||
field="authored_date", min_doc_count=1, **interval_kwargs
|
||||
),
|
||||
"deletions": RangeFacet(
|
||||
field="stats.deletions",
|
||||
ranges=[("ok", (None, 1)), ("good", (1, 5)), ("better", (5, None))],
|
||||
),
|
||||
}
|
||||
|
||||
return CommitSearch
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def repo_search_cls(opensearch_version):
|
||||
interval_type = "calendar_interval"
|
||||
|
||||
class RepoSearch(AsyncFacetedSearch):
|
||||
index = "git"
|
||||
doc_types = [Repos]
|
||||
facets = {
|
||||
"public": TermsFacet(field="is_public"),
|
||||
"created": DateHistogramFacet(
|
||||
field="created_at", **{interval_type: "month"}
|
||||
),
|
||||
}
|
||||
|
||||
def search(self):
|
||||
s = super(RepoSearch, self).search()
|
||||
return s.filter("term", commit_repo="repo")
|
||||
|
||||
return RepoSearch
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def pr_search_cls(opensearch_version):
|
||||
interval_type = "calendar_interval"
|
||||
|
||||
class PRSearch(AsyncFacetedSearch):
|
||||
index = "test-prs"
|
||||
doc_types = [PullRequest]
|
||||
facets = {
|
||||
"comments": NestedFacet(
|
||||
"comments",
|
||||
DateHistogramFacet(
|
||||
field="comments.created_at", **{interval_type: "month"}
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
return PRSearch
|
||||
|
||||
|
||||
async def test_facet_with_custom_metric(data_client):
|
||||
ms = MetricSearch()
|
||||
r = await ms.execute()
|
||||
|
||||
dates = [f[1] for f in r.facets.files]
|
||||
assert dates == list(sorted(dates, reverse=True))
|
||||
assert dates[0] == 1399038439000
|
||||
|
||||
|
||||
async def test_nested_facet(pull_request, pr_search_cls):
|
||||
prs = pr_search_cls()
|
||||
r = await prs.execute()
|
||||
|
||||
assert r.hits.total.value == 1
|
||||
assert [(datetime(2018, 1, 1, 0, 0), 1, False)] == r.facets.comments
|
||||
|
||||
|
||||
async def test_nested_facet_with_filter(pull_request, pr_search_cls):
|
||||
prs = pr_search_cls(filters={"comments": datetime(2018, 1, 1, 0, 0)})
|
||||
r = await prs.execute()
|
||||
|
||||
assert r.hits.total.value == 1
|
||||
assert [(datetime(2018, 1, 1, 0, 0), 1, True)] == r.facets.comments
|
||||
|
||||
prs = pr_search_cls(filters={"comments": datetime(2018, 2, 1, 0, 0)})
|
||||
r = await prs.execute()
|
||||
assert not r.hits
|
||||
|
||||
|
||||
async def test_datehistogram_facet(data_client, repo_search_cls):
|
||||
rs = repo_search_cls()
|
||||
r = await rs.execute()
|
||||
|
||||
assert r.hits.total.value == 1
|
||||
assert [(datetime(2014, 3, 1, 0, 0), 1, False)] == r.facets.created
|
||||
|
||||
|
||||
async def test_boolean_facet(data_client, repo_search_cls):
|
||||
rs = repo_search_cls()
|
||||
r = await rs.execute()
|
||||
|
||||
assert r.hits.total.value == 1
|
||||
assert [(True, 1, False)] == r.facets.public
|
||||
value, count, selected = r.facets.public[0]
|
||||
assert value is True
|
||||
|
||||
|
||||
async def test_empty_search_finds_everything(
|
||||
data_client, opensearch_version, commit_search_cls
|
||||
):
|
||||
cs = commit_search_cls()
|
||||
r = await cs.execute()
|
||||
assert r.hits.total.value == 52
|
||||
assert [
|
||||
("opensearchpy", 39, False),
|
||||
("test_opensearchpy", 35, False),
|
||||
("test_opensearchpy/test_dsl", 35, False),
|
||||
("opensearchpy/query.py", 18, False),
|
||||
("test_opensearchpy/test_dsl/test_search.py", 15, False),
|
||||
("opensearchpy/utils.py", 14, False),
|
||||
("test_opensearchpy/test_dsl/test_query.py", 13, False),
|
||||
("opensearchpy/search.py", 12, False),
|
||||
("opensearchpy/aggs.py", 11, False),
|
||||
("test_opensearchpy/test_dsl/test_result.py", 5, False),
|
||||
] == r.facets.files
|
||||
|
||||
assert [
|
||||
(datetime(2014, 3, 3, 0, 0), 2, False),
|
||||
(datetime(2014, 3, 4, 0, 0), 1, False),
|
||||
(datetime(2014, 3, 5, 0, 0), 3, False),
|
||||
(datetime(2014, 3, 6, 0, 0), 3, False),
|
||||
(datetime(2014, 3, 7, 0, 0), 9, False),
|
||||
(datetime(2014, 3, 10, 0, 0), 2, False),
|
||||
(datetime(2014, 3, 15, 0, 0), 4, False),
|
||||
(datetime(2014, 3, 21, 0, 0), 2, False),
|
||||
(datetime(2014, 3, 23, 0, 0), 2, False),
|
||||
(datetime(2014, 3, 24, 0, 0), 10, False),
|
||||
(datetime(2014, 4, 20, 0, 0), 2, False),
|
||||
(datetime(2014, 4, 22, 0, 0), 2, False),
|
||||
(datetime(2014, 4, 25, 0, 0), 3, False),
|
||||
(datetime(2014, 4, 26, 0, 0), 2, False),
|
||||
(datetime(2014, 4, 27, 0, 0), 2, False),
|
||||
(datetime(2014, 5, 1, 0, 0), 2, False),
|
||||
(datetime(2014, 5, 2, 0, 0), 1, False),
|
||||
] == r.facets.frequency
|
||||
|
||||
assert [
|
||||
("ok", 19, False),
|
||||
("good", 14, False),
|
||||
("better", 19, False),
|
||||
] == r.facets.deletions
|
||||
|
||||
|
||||
async def test_term_filters_are_shown_as_selected_and_data_is_filtered(
|
||||
data_client, commit_search_cls
|
||||
):
|
||||
cs = commit_search_cls(filters={"files": "test_opensearchpy/test_dsl"})
|
||||
|
||||
r = await cs.execute()
|
||||
|
||||
assert 35 == r.hits.total.value
|
||||
assert [
|
||||
("opensearchpy", 39, False),
|
||||
("test_opensearchpy", 35, False),
|
||||
("test_opensearchpy/test_dsl", 35, True),
|
||||
("opensearchpy/query.py", 18, False),
|
||||
("test_opensearchpy/test_dsl/test_search.py", 15, False),
|
||||
("opensearchpy/utils.py", 14, False),
|
||||
("test_opensearchpy/test_dsl/test_query.py", 13, False),
|
||||
("opensearchpy/search.py", 12, False),
|
||||
("opensearchpy/aggs.py", 11, False),
|
||||
("test_opensearchpy/test_dsl/test_result.py", 5, False),
|
||||
] == r.facets.files
|
||||
|
||||
assert [
|
||||
(datetime(2014, 3, 3, 0, 0), 1, False),
|
||||
(datetime(2014, 3, 5, 0, 0), 2, False),
|
||||
(datetime(2014, 3, 6, 0, 0), 3, False),
|
||||
(datetime(2014, 3, 7, 0, 0), 6, False),
|
||||
(datetime(2014, 3, 10, 0, 0), 1, False),
|
||||
(datetime(2014, 3, 15, 0, 0), 3, False),
|
||||
(datetime(2014, 3, 21, 0, 0), 2, False),
|
||||
(datetime(2014, 3, 23, 0, 0), 1, False),
|
||||
(datetime(2014, 3, 24, 0, 0), 7, False),
|
||||
(datetime(2014, 4, 20, 0, 0), 1, False),
|
||||
(datetime(2014, 4, 25, 0, 0), 3, False),
|
||||
(datetime(2014, 4, 26, 0, 0), 2, False),
|
||||
(datetime(2014, 4, 27, 0, 0), 1, False),
|
||||
(datetime(2014, 5, 1, 0, 0), 1, False),
|
||||
(datetime(2014, 5, 2, 0, 0), 1, False),
|
||||
] == r.facets.frequency
|
||||
|
||||
assert [
|
||||
("ok", 12, False),
|
||||
("good", 10, False),
|
||||
("better", 13, False),
|
||||
] == r.facets.deletions
|
||||
|
||||
|
||||
async def test_range_filters_are_shown_as_selected_and_data_is_filtered(
|
||||
data_client, commit_search_cls
|
||||
):
|
||||
cs = commit_search_cls(filters={"deletions": "better"})
|
||||
|
||||
r = await cs.execute()
|
||||
|
||||
assert 19 == r.hits.total.value
|
||||
|
||||
|
||||
async def test_pagination(data_client, commit_search_cls):
|
||||
cs = commit_search_cls()
|
||||
cs = cs[0:20]
|
||||
|
||||
assert 52 == await cs.count()
|
||||
assert 20 == len(await cs.execute())
|
||||
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
|
||||
import pytest
|
||||
|
||||
from opensearchpy import Date, Text
|
||||
from opensearchpy._async.helpers.document import AsyncDocument
|
||||
from opensearchpy._async.helpers.index import AsyncIndex, AsyncIndexTemplate
|
||||
from opensearchpy.helpers import analysis
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
class Post(AsyncDocument):
|
||||
title = Text(analyzer=analysis.analyzer("my_analyzer", tokenizer="keyword"))
|
||||
published_from = Date()
|
||||
|
||||
|
||||
async def test_index_template_works(write_client):
|
||||
it = AsyncIndexTemplate("test-template", "test-*")
|
||||
it.document(Post)
|
||||
it.settings(number_of_replicas=0, number_of_shards=1)
|
||||
await it.save()
|
||||
|
||||
i = AsyncIndex("test-blog")
|
||||
await i.create()
|
||||
|
||||
assert {
|
||||
"test-blog": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"title": {"type": "text", "analyzer": "my_analyzer"},
|
||||
"published_from": {"type": "date"},
|
||||
}
|
||||
}
|
||||
}
|
||||
} == await write_client.indices.get_mapping(index="test-blog")
|
||||
|
||||
|
||||
async def test_index_can_be_saved_even_with_settings(write_client):
|
||||
i = AsyncIndex("test-blog", using=write_client)
|
||||
i.settings(number_of_shards=3, number_of_replicas=0)
|
||||
await i.save()
|
||||
i.settings(number_of_replicas=1)
|
||||
await i.save()
|
||||
|
||||
assert (
|
||||
"1"
|
||||
== (await i.get_settings())["test-blog"]["settings"]["index"][
|
||||
"number_of_replicas"
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def test_index_exists(data_client):
|
||||
assert await AsyncIndex("git").exists()
|
||||
assert not await AsyncIndex("not-there").exists()
|
||||
|
||||
|
||||
async def test_index_can_be_created_with_settings_and_mappings(write_client):
|
||||
i = AsyncIndex("test-blog", using=write_client)
|
||||
i.document(Post)
|
||||
i.settings(number_of_replicas=0, number_of_shards=1)
|
||||
await i.create()
|
||||
|
||||
assert {
|
||||
"test-blog": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"title": {"type": "text", "analyzer": "my_analyzer"},
|
||||
"published_from": {"type": "date"},
|
||||
}
|
||||
}
|
||||
}
|
||||
} == await write_client.indices.get_mapping(index="test-blog")
|
||||
|
||||
settings = await write_client.indices.get_settings(index="test-blog")
|
||||
assert settings["test-blog"]["settings"]["index"]["number_of_replicas"] == "0"
|
||||
assert settings["test-blog"]["settings"]["index"]["number_of_shards"] == "1"
|
||||
assert settings["test-blog"]["settings"]["index"]["analysis"] == {
|
||||
"analyzer": {"my_analyzer": {"type": "custom", "tokenizer": "keyword"}}
|
||||
}
|
||||
|
||||
|
||||
async def test_delete(write_client):
|
||||
await write_client.indices.create(
|
||||
index="test-index",
|
||||
body={"settings": {"number_of_replicas": 0, "number_of_shards": 1}},
|
||||
)
|
||||
|
||||
i = AsyncIndex("test-index", using=write_client)
|
||||
await i.delete()
|
||||
assert not await write_client.indices.exists(index="test-index")
|
||||
|
||||
|
||||
async def test_multiple_indices_with_same_doc_type_work(write_client):
|
||||
i1 = AsyncIndex("test-index-1", using=write_client)
|
||||
i2 = AsyncIndex("test-index-2", using=write_client)
|
||||
|
||||
for i in i1, i2:
|
||||
i.document(Post)
|
||||
await i.create()
|
||||
|
||||
for i in ("test-index-1", "test-index-2"):
|
||||
settings = await write_client.indices.get_settings(index=i)
|
||||
assert settings[i]["settings"]["index"]["analysis"] == {
|
||||
"analyzer": {"my_analyzer": {"type": "custom", "tokenizer": "keyword"}}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
|
||||
import pytest
|
||||
from pytest import raises
|
||||
|
||||
from opensearchpy import exceptions
|
||||
from opensearchpy._async.helpers import mapping
|
||||
from opensearchpy.helpers import analysis
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_mapping_saved_into_opensearch(write_client):
|
||||
m = mapping.AsyncMapping()
|
||||
m.field(
|
||||
"name", "text", analyzer=analysis.analyzer("my_analyzer", tokenizer="keyword")
|
||||
)
|
||||
m.field("tags", "keyword")
|
||||
await m.save("test-mapping", using=write_client)
|
||||
|
||||
assert {
|
||||
"test-mapping": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"name": {"type": "text", "analyzer": "my_analyzer"},
|
||||
"tags": {"type": "keyword"},
|
||||
}
|
||||
}
|
||||
}
|
||||
} == await write_client.indices.get_mapping(index="test-mapping")
|
||||
|
||||
|
||||
async def test_mapping_saved_into_opensearch_when_index_already_exists_closed(
|
||||
write_client,
|
||||
):
|
||||
m = mapping.AsyncMapping()
|
||||
m.field(
|
||||
"name", "text", analyzer=analysis.analyzer("my_analyzer", tokenizer="keyword")
|
||||
)
|
||||
await write_client.indices.create(index="test-mapping")
|
||||
|
||||
with raises(exceptions.IllegalOperation):
|
||||
await m.save("test-mapping", using=write_client)
|
||||
|
||||
await write_client.cluster.health(index="test-mapping", wait_for_status="yellow")
|
||||
await write_client.indices.close(index="test-mapping")
|
||||
await m.save("test-mapping", using=write_client)
|
||||
|
||||
assert {
|
||||
"test-mapping": {
|
||||
"mappings": {
|
||||
"properties": {"name": {"type": "text", "analyzer": "my_analyzer"}}
|
||||
}
|
||||
}
|
||||
} == await write_client.indices.get_mapping(index="test-mapping")
|
||||
|
||||
|
||||
async def test_mapping_saved_into_opensearch_when_index_already_exists_with_analysis(
|
||||
write_client,
|
||||
):
|
||||
m = mapping.AsyncMapping()
|
||||
analyzer = analysis.analyzer("my_analyzer", tokenizer="keyword")
|
||||
m.field("name", "text", analyzer=analyzer)
|
||||
|
||||
new_analysis = analyzer.get_analysis_definition()
|
||||
new_analysis["analyzer"]["other_analyzer"] = {
|
||||
"type": "custom",
|
||||
"tokenizer": "whitespace",
|
||||
}
|
||||
await write_client.indices.create(
|
||||
index="test-mapping", body={"settings": {"analysis": new_analysis}}
|
||||
)
|
||||
|
||||
m.field("title", "text", analyzer=analyzer)
|
||||
await m.save("test-mapping", using=write_client)
|
||||
|
||||
assert {
|
||||
"test-mapping": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"name": {"type": "text", "analyzer": "my_analyzer"},
|
||||
"title": {"type": "text", "analyzer": "my_analyzer"},
|
||||
}
|
||||
}
|
||||
}
|
||||
} == await write_client.indices.get_mapping(index="test-mapping")
|
||||
|
||||
|
||||
async def test_mapping_gets_updated_from_opensearch(write_client):
|
||||
await write_client.indices.create(
|
||||
index="test-mapping",
|
||||
body={
|
||||
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
|
||||
"mappings": {
|
||||
"date_detection": False,
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "text",
|
||||
"analyzer": "snowball",
|
||||
"fields": {"raw": {"type": "keyword"}},
|
||||
},
|
||||
"created_at": {"type": "date"},
|
||||
"comments": {
|
||||
"type": "nested",
|
||||
"properties": {
|
||||
"created": {"type": "date"},
|
||||
"author": {
|
||||
"type": "text",
|
||||
"analyzer": "snowball",
|
||||
"fields": {"raw": {"type": "keyword"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
m = await mapping.AsyncMapping.from_opensearch("test-mapping", using=write_client)
|
||||
|
||||
assert ["comments", "created_at", "title"] == list(
|
||||
sorted(m.properties.properties._d_.keys())
|
||||
)
|
||||
assert {
|
||||
"date_detection": False,
|
||||
"properties": {
|
||||
"comments": {
|
||||
"type": "nested",
|
||||
"properties": {
|
||||
"created": {"type": "date"},
|
||||
"author": {
|
||||
"analyzer": "snowball",
|
||||
"fields": {"raw": {"type": "keyword"}},
|
||||
"type": "text",
|
||||
},
|
||||
},
|
||||
},
|
||||
"created_at": {"type": "date"},
|
||||
"title": {
|
||||
"analyzer": "snowball",
|
||||
"fields": {"raw": {"type": "keyword"}},
|
||||
"type": "text",
|
||||
},
|
||||
},
|
||||
} == m.to_dict()
|
||||
|
||||
# test same with alias
|
||||
await write_client.indices.put_alias(index="test-mapping", name="test-alias")
|
||||
|
||||
m2 = await mapping.AsyncMapping.from_opensearch("test-alias", using=write_client)
|
||||
assert m2.to_dict() == m.to_dict()
|
||||
@@ -0,0 +1,161 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import pytest
|
||||
from pytest import raises
|
||||
|
||||
from opensearchpy import Date, Keyword, Q, Text, TransportError
|
||||
from opensearchpy._async.helpers.document import AsyncDocument
|
||||
from opensearchpy._async.helpers.search import AsyncMultiSearch, AsyncSearch
|
||||
from opensearchpy.helpers.response import aggs
|
||||
from test_opensearchpy.test_server.test_helpers.test_data import FLAT_DATA
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
class Repository(AsyncDocument):
|
||||
created_at = Date()
|
||||
description = Text(analyzer="snowball")
|
||||
tags = Keyword()
|
||||
|
||||
@classmethod
|
||||
def search(cls):
|
||||
return super(Repository, cls).search().filter("term", commit_repo="repo")
|
||||
|
||||
class Index:
|
||||
name = "git"
|
||||
|
||||
|
||||
class Commit(AsyncDocument):
|
||||
class Index:
|
||||
name = "flat-git"
|
||||
|
||||
|
||||
async def test_filters_aggregation_buckets_are_accessible(data_client):
|
||||
has_tests_query = Q("term", files="test_opensearchpy/test_dsl")
|
||||
s = Commit.search()[0:0]
|
||||
s.aggs.bucket("top_authors", "terms", field="author.name.raw").bucket(
|
||||
"has_tests", "filters", filters={"yes": has_tests_query, "no": ~has_tests_query}
|
||||
).metric("lines", "stats", field="stats.lines")
|
||||
response = await s.execute()
|
||||
|
||||
assert isinstance(
|
||||
response.aggregations.top_authors.buckets[0].has_tests.buckets.yes, aggs.Bucket
|
||||
)
|
||||
assert (
|
||||
35
|
||||
== response.aggregations.top_authors.buckets[0].has_tests.buckets.yes.doc_count
|
||||
)
|
||||
assert (
|
||||
228
|
||||
== response.aggregations.top_authors.buckets[0].has_tests.buckets.yes.lines.max
|
||||
)
|
||||
|
||||
|
||||
async def test_top_hits_are_wrapped_in_response(data_client):
|
||||
s = Commit.search()[0:0]
|
||||
s.aggs.bucket("top_authors", "terms", field="author.name.raw").metric(
|
||||
"top_commits", "top_hits", size=5
|
||||
)
|
||||
response = await s.execute()
|
||||
|
||||
top_commits = response.aggregations.top_authors.buckets[0].top_commits
|
||||
assert isinstance(top_commits, aggs.TopHitsData)
|
||||
assert 5 == len(top_commits)
|
||||
|
||||
hits = [h for h in top_commits]
|
||||
assert 5 == len(hits)
|
||||
assert isinstance(hits[0], Commit)
|
||||
|
||||
|
||||
async def test_inner_hits_are_wrapped_in_response(data_client):
|
||||
s = AsyncSearch(index="git")[0:1].query(
|
||||
"has_parent", parent_type="repo", inner_hits={}, query=Q("match_all")
|
||||
)
|
||||
response = await s.execute()
|
||||
|
||||
commit = response.hits[0]
|
||||
assert isinstance(commit.meta.inner_hits.repo, response.__class__)
|
||||
assert repr(commit.meta.inner_hits.repo[0]).startswith("<Hit(git/opensearch-py): ")
|
||||
|
||||
|
||||
async def test_scan_respects_doc_types(data_client):
|
||||
result = Repository.search().scan()
|
||||
repos = await get_result(result)
|
||||
|
||||
assert 1 == len(repos)
|
||||
assert isinstance(repos[0], Repository)
|
||||
assert repos[0].organization == "opensearch"
|
||||
|
||||
|
||||
async def test_scan_iterates_through_all_docs(data_client):
|
||||
s = AsyncSearch(index="flat-git")
|
||||
result = s.scan()
|
||||
commits = await get_result(result)
|
||||
|
||||
assert 52 == len(commits)
|
||||
assert {d["_id"] for d in FLAT_DATA} == {c.meta.id for c in commits}
|
||||
|
||||
|
||||
async def get_result(b):
|
||||
a = []
|
||||
async for i in b:
|
||||
a.append(i)
|
||||
return a
|
||||
|
||||
|
||||
async def test_multi_search(data_client):
|
||||
s1 = Repository.search()
|
||||
s2 = AsyncSearch(index="flat-git")
|
||||
|
||||
ms = AsyncMultiSearch()
|
||||
ms = ms.add(s1).add(s2)
|
||||
|
||||
r1, r2 = await ms.execute()
|
||||
|
||||
assert 1 == len(r1)
|
||||
assert isinstance(r1[0], Repository)
|
||||
assert r1._search is s1
|
||||
|
||||
assert 52 == r2.hits.total.value
|
||||
assert r2._search is s2
|
||||
|
||||
|
||||
async def test_multi_missing(data_client):
|
||||
s1 = Repository.search()
|
||||
s2 = AsyncSearch(index="flat-git")
|
||||
s3 = AsyncSearch(index="does_not_exist")
|
||||
|
||||
ms = AsyncMultiSearch()
|
||||
ms = ms.add(s1).add(s2).add(s3)
|
||||
|
||||
with raises(TransportError):
|
||||
await ms.execute()
|
||||
|
||||
r1, r2, r3 = await ms.execute(raise_on_error=False)
|
||||
|
||||
assert 1 == len(r1)
|
||||
assert isinstance(r1[0], Repository)
|
||||
assert r1._search is s1
|
||||
|
||||
assert 52 == r2.hits.total.value
|
||||
assert r2._search is s2
|
||||
|
||||
assert r3 is None
|
||||
|
||||
|
||||
async def test_raw_subfield_can_be_used_in_aggs(data_client):
|
||||
s = AsyncSearch(index="git")[0:0]
|
||||
s.aggs.bucket("authors", "terms", field="author.name.raw", size=1)
|
||||
r = await s.execute()
|
||||
authors = r.aggregations.authors
|
||||
assert 1 == len(authors)
|
||||
assert {"key": "Honza Král", "doc_count": 52} == authors[0]
|
||||
@@ -0,0 +1,69 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The OpenSearch Contributors require contributions made to
|
||||
# this file be licensed under the Apache-2.0 license or a
|
||||
# compatible open source license.
|
||||
#
|
||||
# Modifications Copyright OpenSearch Contributors. See
|
||||
# GitHub history for details.
|
||||
|
||||
import pytest
|
||||
|
||||
from opensearchpy._async.helpers.update_by_query import AsyncUpdateByQuery
|
||||
from opensearchpy.helpers.search import Q
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_update_by_query_no_script(write_client, setup_ubq_tests):
|
||||
index = setup_ubq_tests
|
||||
|
||||
ubq = (
|
||||
AsyncUpdateByQuery(using=write_client)
|
||||
.index(index)
|
||||
.filter(~Q("exists", field="is_public"))
|
||||
)
|
||||
response = await ubq.execute()
|
||||
|
||||
assert response.total == 52
|
||||
assert response["took"] > 0
|
||||
assert not response.timed_out
|
||||
assert response.updated == 52
|
||||
assert response.deleted == 0
|
||||
assert response.took > 0
|
||||
assert response.success()
|
||||
|
||||
|
||||
async def test_update_by_query_with_script(write_client, setup_ubq_tests):
|
||||
index = setup_ubq_tests
|
||||
|
||||
ubq = (
|
||||
AsyncUpdateByQuery(using=write_client)
|
||||
.index(index)
|
||||
.filter(~Q("exists", field="parent_shas"))
|
||||
.script(source="ctx._source.is_public = false")
|
||||
)
|
||||
ubq = ubq.params(conflicts="proceed")
|
||||
|
||||
response = await ubq.execute()
|
||||
assert response.total == 2
|
||||
assert response.updated == 2
|
||||
assert response.version_conflicts == 0
|
||||
|
||||
|
||||
async def test_delete_by_query_with_script(write_client, setup_ubq_tests):
|
||||
index = setup_ubq_tests
|
||||
|
||||
ubq = (
|
||||
AsyncUpdateByQuery(using=write_client)
|
||||
.index(index)
|
||||
.filter(Q("match", parent_shas="1dd19210b5be92b960f7db6f66ae526288edccc3"))
|
||||
.script(source='ctx.op = "delete"')
|
||||
)
|
||||
ubq = ubq.params(conflicts="proceed")
|
||||
|
||||
response = await ubq.execute()
|
||||
|
||||
assert response.total == 1
|
||||
assert response.deleted == 1
|
||||
assert response.success()
|
||||
Reference in New Issue
Block a user