Make sure scan helper cleans up after itself

Closes #389
This commit is contained in:
Honza Král
2016-05-17 15:14:27 +02:00
parent 7c305f1bc3
commit 1ce17950e0
+8 -1
View File
@@ -16,7 +16,9 @@ class BulkIndexError(ElasticsearchException):
class ScanError(ElasticsearchException): class ScanError(ElasticsearchException):
pass def __init__(self, scroll_id, *args, **kwargs):
super(ScanError, self).__init__(*args, **kwargs)
self.scroll_id = scroll_id
def expand_action(data): def expand_action(data):
""" """
@@ -277,6 +279,7 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
if scroll_id is None: if scroll_id is None:
return return
try:
first_run = True first_run = True
while True: while True:
# if we didn't set search_type to scan initial search contains data # if we didn't set search_type to scan initial search contains data
@@ -296,6 +299,7 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
) )
if raise_on_error: if raise_on_error:
raise ScanError( raise ScanError(
scroll_id,
'Scroll request has failed on %d shards out of %d.' % 'Scroll request has failed on %d shards out of %d.' %
(resp['_shards']['failed'], resp['_shards']['total']) (resp['_shards']['failed'], resp['_shards']['total'])
) )
@@ -304,6 +308,9 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
# end of scroll # end of scroll
if scroll_id is None or not resp['hits']['hits']: if scroll_id is None or not resp['hits']['hits']:
break break
finally:
if scroll_id:
client.clear_scroll(body={'scroll_id': [scroll_id]}, ignore=(404, ))
def reindex(client, source_index, target_index, query=None, target_client=None, def reindex(client, source_index, target_index, query=None, target_client=None,
chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}): chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}):