Fix: TypeError on calling parallel_bulk. (#601)

* Fix: TypeError on calling parallel_bulk. Signed-off-by: dblock <[email protected]> * Added a sample that uses a bulk function generator. Signed-off-by: dblock <[email protected]> --------- Signed-off-by: dblock <[email protected]>
2023-11-17 10:18:42 -08:00
parent 567ede3023
commit c66e1e37be
5 changed files with 172 additions and 3 deletions
@@ -12,6 +12,7 @@


 import os
+from typing import Any

 from opensearchpy import OpenSearch, helpers

@@ -49,8 +50,57 @@ data = []
 for i in range(100):
    data.append({"_index": index_name, "_id": i, "value": i})

+# serialized bulk raising an exception on error
 rc = helpers.bulk(client, data)
-print(f"Bulk-inserted {rc[0]} items.")
+print(f"Bulk-inserted {rc[0]} items (bulk).")
+
+# parallel bulk with explicit error checking
+succeeded = []
+failed = []
+for success, item in helpers.parallel_bulk(
+    client,
+    actions=data,
+    chunk_size=10,
+    raise_on_error=False,
+    raise_on_exception=False,
+    max_chunk_bytes=20 * 1024 * 1024,
+    request_timeout=60,
+):
+    if success:
+        succeeded.append(item)
+    else:
+        failed.append(item)
+
+if len(failed) > 0:
+    print(f"There were {len(failed)} errors:")
+    for item in failed:
+        print(item["index"]["error"])
+
+if len(succeeded) > 0:
+    print(f"Bulk-inserted {len(succeeded)} items (parallel_bulk).")
+
+
+# streaming bulk with a data generator
+def _generate_data() -> Any:
+    for i in range(100):
+        yield {"_index": index_name, "_id": i, "value": i}
+
+
+succeeded = []
+failed = []
+for success, item in helpers.streaming_bulk(client, actions=_generate_data()):
+    if success:
+        succeeded.append(item)
+    else:
+        failed.append(item)
+
+if len(failed) > 0:
+    print(f"There were {len(failed)} errors:")
+    for item in failed:
+        print(item["index"]["error"])
+
+if len(succeeded) > 0:
+    print(f"Bulk-inserted {len(succeeded)} items (streaming_bulk).")

 # delete index
 client.indices.delete(index=index_name)