Make sure scan helper cleans up after itself

Closes #389
This commit is contained in:
Honza Král
2016-05-17 15:14:27 +02:00
parent 7c305f1bc3
commit 1ce17950e0
+8 -1
View File
@@ -16,7 +16,9 @@ class BulkIndexError(ElasticsearchException):
class ScanError(ElasticsearchException):
pass
def __init__(self, scroll_id, *args, **kwargs):
super(ScanError, self).__init__(*args, **kwargs)
self.scroll_id = scroll_id
def expand_action(data):
"""
@@ -277,6 +279,7 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
if scroll_id is None:
return
try:
first_run = True
while True:
# if we didn't set search_type to scan initial search contains data
@@ -296,6 +299,7 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
)
if raise_on_error:
raise ScanError(
scroll_id,
'Scroll request has failed on %d shards out of %d.' %
(resp['_shards']['failed'], resp['_shards']['total'])
)
@@ -304,6 +308,9 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
# end of scroll
if scroll_id is None or not resp['hits']['hits']:
break
finally:
if scroll_id:
client.clear_scroll(body={'scroll_id': [scroll_id]}, ignore=(404, ))
def reindex(client, source_index, target_index, query=None, target_client=None,
chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}):