@@ -16,7 +16,9 @@ class BulkIndexError(ElasticsearchException):
|
|||||||
|
|
||||||
|
|
||||||
class ScanError(ElasticsearchException):
|
class ScanError(ElasticsearchException):
|
||||||
pass
|
def __init__(self, scroll_id, *args, **kwargs):
|
||||||
|
super(ScanError, self).__init__(*args, **kwargs)
|
||||||
|
self.scroll_id = scroll_id
|
||||||
|
|
||||||
def expand_action(data):
|
def expand_action(data):
|
||||||
"""
|
"""
|
||||||
@@ -277,6 +279,7 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
|
|||||||
if scroll_id is None:
|
if scroll_id is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
first_run = True
|
first_run = True
|
||||||
while True:
|
while True:
|
||||||
# if we didn't set search_type to scan initial search contains data
|
# if we didn't set search_type to scan initial search contains data
|
||||||
@@ -296,6 +299,7 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
|
|||||||
)
|
)
|
||||||
if raise_on_error:
|
if raise_on_error:
|
||||||
raise ScanError(
|
raise ScanError(
|
||||||
|
scroll_id,
|
||||||
'Scroll request has failed on %d shards out of %d.' %
|
'Scroll request has failed on %d shards out of %d.' %
|
||||||
(resp['_shards']['failed'], resp['_shards']['total'])
|
(resp['_shards']['failed'], resp['_shards']['total'])
|
||||||
)
|
)
|
||||||
@@ -304,6 +308,9 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=Fa
|
|||||||
# end of scroll
|
# end of scroll
|
||||||
if scroll_id is None or not resp['hits']['hits']:
|
if scroll_id is None or not resp['hits']['hits']:
|
||||||
break
|
break
|
||||||
|
finally:
|
||||||
|
if scroll_id:
|
||||||
|
client.clear_scroll(body={'scroll_id': [scroll_id]}, ignore=(404, ))
|
||||||
|
|
||||||
def reindex(client, source_index, target_index, query=None, target_client=None,
|
def reindex(client, source_index, target_index, query=None, target_client=None,
|
||||||
chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}):
|
chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}):
|
||||||
|
|||||||
Reference in New Issue
Block a user