actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[7.x] Allow '_shards.skipped' to be absent on scroll responses

Browse Source 
Co-authored-by: Seth Michael Larson <[email protected]>
This commit is contained in:
github-actions[bot]
2020-11-20 16:41:58 -06:00
committed by GitHub
co-authored by Seth Michael Larson
parent a1cac1aaad
commit bbb72cd932
3 changed files with 57 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
elasticsearch/_async/helpers.py
+15 -11
View File
@@ -336,24 +336,28 @@ async def async_scan(
for hit in resp["hits"]["hits"]:
yield hit
# Default to 0 if the value isn't included in the response
shards_successful = resp["_shards"].get("successful", 0)
shards_skipped = resp["_shards"].get("skipped", 0)
shards_total = resp["_shards"].get("total", 0)
# check if we have any errors
if (resp["_shards"]["successful"] + resp["_shards"]["skipped"]) < resp[
"_shards"
]["total"]:
if (shards_successful + shards_skipped) < shards_total:
shards_message = "Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
logger.warning(
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d.",
resp["_shards"]["successful"],
resp["_shards"]["skipped"],
resp["_shards"]["total"],
shards_message,
shards_successful,
shards_skipped,
shards_total,
)
if raise_on_error:
raise ScanError(
scroll_id,
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
shards_message
% (
resp["_shards"]["successful"],
resp["_shards"]["skipped"],
resp["_shards"]["total"],
shards_successful,
shards_skipped,
shards_total,
),
)
resp = await client.scroll(
elasticsearch/helpers/actions.py
+15 -11
View File
@@ -531,24 +531,28 @@ def scan(
for hit in resp["hits"]["hits"]:
yield hit
# Default to 0 if the value isn't included in the response
shards_successful = resp["_shards"].get("successful", 0)
shards_skipped = resp["_shards"].get("skipped", 0)
shards_total = resp["_shards"].get("total", 0)
# check if we have any errors
if (resp["_shards"]["successful"] + resp["_shards"]["skipped"]) < resp[
"_shards"
]["total"]:
if (shards_successful + shards_skipped) < shards_total:
shards_message = "Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
logger.warning(
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d.",
resp["_shards"]["successful"],
resp["_shards"]["skipped"],
resp["_shards"]["total"],
shards_message,
shards_successful,
shards_skipped,
shards_total,
)
if raise_on_error:
raise ScanError(
scroll_id,
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
shards_message
% (
resp["_shards"]["successful"],
resp["_shards"]["skipped"],
resp["_shards"]["total"],
shards_successful,
shards_skipped,
shards_total,
),
)
resp = client.scroll(
test_elasticsearch/test_server/test_helpers.py
+27
View File
@@ -508,6 +508,33 @@ class TestScan(ElasticsearchTestCase):
)
spy.assert_not_called()
def test_shards_no_skipped_field(self):
with patch.object(self, "client") as client_mock:
client_mock.search.return_value = {
"_scroll_id": "dummy_id",
"_shards": {"successful": 5, "total": 5},
"hits": {"hits": [{"search_data": 1}]},
}
client_mock.scroll.side_effect = [
{
"_scroll_id": "dummy_id",
"_shards": {"successful": 5, "total": 5},
"hits": {"hits": [{"scroll_data": 42}]},
},
{
"_scroll_id": "dummy_id",
"_shards": {"successful": 5, "total": 5},
"hits": {"hits": []},
},
]
data = list(
helpers.scan(
self.client, index="test_index", size=2, raise_on_error=True
)
)
self.assertEqual(data, [{"search_data": 1}, {"scroll_data": 42}])
class TestReindex(ElasticsearchTestCase):
def setup_method(self, _):