actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Extract chunking from streaming_bulk

Browse Source
This commit is contained in:
Honza Král
2015-09-30 18:23:42 +02:00
parent 14f5b35a23
commit 0ad6b94d1e
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
elasticsearch/helpers/__init__.py
+4 -2
View File
@@ -38,6 +38,9 @@ def expand_action(data):
return action, data.get('_source', data) return action, data.get('_source', data)
def _chunk_actions(actions, chunk_size):
while True:
yield islice(actions, chunk_size)
def streaming_bulk(client, actions, chunk_size=500, raise_on_error=True, def streaming_bulk(client, actions, chunk_size=500, raise_on_error=True,
expand_action_callback=expand_action, raise_on_exception=True, expand_action_callback=expand_action, raise_on_exception=True,
@@ -100,8 +103,7 @@ def streaming_bulk(client, actions, chunk_size=500, raise_on_error=True,
# if raise on error is set, we need to collect errors per chunk before raising them # if raise on error is set, we need to collect errors per chunk before raising them
errors = [] errors = []
while True: for chunk in _chunk_actions(actions, chunk_size):
chunk = islice(actions, chunk_size)
# raise on exception means we might need to iterate on chunk twice # raise on exception means we might need to iterate on chunk twice
if not raise_on_exception: if not raise_on_exception: