Make sure we always close the pool when doing parallel_bulk

This commit is contained in:
Honza Král
2015-12-09 18:35:14 +01:00
parent 74cb686538
commit b62d950bb8
+10 -8
View File
@@ -222,15 +222,17 @@ def parallel_bulk(client, actions, thread_count=4, chunk_size=500,
pool = Pool(thread_count)
for result in pool.imap(
lambda chunk: list(_process_bulk_chunk(client, chunk, **kwargs)),
_chunk_actions(actions, chunk_size, max_chunk_bytes, client.transport.serializer)
):
for item in result:
yield item
try:
for result in pool.imap(
lambda chunk: list(_process_bulk_chunk(client, chunk, **kwargs)),
_chunk_actions(actions, chunk_size, max_chunk_bytes, client.transport.serializer)
):
for item in result:
yield item
pool.close()
pool.join()
finally:
pool.close()
pool.join()
def scan(client, query=None, scroll='5m', raise_on_error=True, preserve_order=False, **kwargs):
"""