actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Make sure scan helper cleans up after itself

Browse Source 
Closes #389
This commit is contained in:
Honza Král
2016-05-17 15:14:27 +02:00
parent 7c305f1bc3
commit 1ce17950e0
1 changed files with 31 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
elasticsearch/helpers/__init__.py
+31 -24
View File
@@ -16,7 +16,9 @@ class BulkIndexError(ElasticsearchException):
class ScanError(ElasticsearchException): class ScanError(ElasticsearchException):
pass def __init__(self, scroll_id, *args, **kwargs):
super(ScanError, self).__init__(*args, **kwargs)
self.scroll_id = scroll_id
def expand_action(data): def expand_action(data):
""" """
@@ -277,33 +279,38 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
if scroll_id is None: if scroll_id is None:
return return
first_run = True try:
while True: first_run = True
# if we didn't set search_type to scan initial search contains data while True:
if first_run: # if we didn't set search_type to scan initial search contains data
first_run = False if first_run:
else: first_run = False
resp = client.scroll(scroll_id, scroll=scroll) else:
resp = client.scroll(scroll_id, scroll=scroll)
for hit in resp['hits']['hits']: for hit in resp['hits']['hits']:
yield hit yield hit
# check if we have any errrors # check if we have any errrors
if resp["_shards"]["failed"]: if resp["_shards"]["failed"]:
logger.warning( logger.warning(
'Scroll request has failed on %d shards out of %d.', 'Scroll request has failed on %d shards out of %d.',
resp['_shards']['failed'], resp['_shards']['total'] resp['_shards']['failed'], resp['_shards']['total']
)
if raise_on_error:
raise ScanError(
'Scroll request has failed on %d shards out of %d.' %
(resp['_shards']['failed'], resp['_shards']['total'])
) )
if raise_on_error:
raise ScanError(
scroll_id,
'Scroll request has failed on %d shards out of %d.' %
(resp['_shards']['failed'], resp['_shards']['total'])
)
scroll_id = resp.get('_scroll_id') scroll_id = resp.get('_scroll_id')
# end of scroll # end of scroll
if scroll_id is None or not resp['hits']['hits']: if scroll_id is None or not resp['hits']['hits']:
break break
finally:
if scroll_id:
client.clear_scroll(body={'scroll_id': [scroll_id]}, ignore=(404, ))
def reindex(client, source_index, target_index, query=None, target_client=None, def reindex(client, source_index, target_index, query=None, target_client=None,
chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}): chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}):