Allow people to just pass raw json strings to bulk helpers
This should allow for significant speedups when indexing json documents from a file
This commit is contained in:
@@ -4,6 +4,8 @@ import logging
|
|||||||
from multiprocessing.dummy import Pool
|
from multiprocessing.dummy import Pool
|
||||||
from operator import methodcaller
|
from operator import methodcaller
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
from ..exceptions import ElasticsearchException, TransportError
|
from ..exceptions import ElasticsearchException, TransportError
|
||||||
from ..compat import map
|
from ..compat import map
|
||||||
|
|
||||||
@@ -25,6 +27,10 @@ def expand_action(data):
|
|||||||
action/data lines needed for elasticsearch's
|
action/data lines needed for elasticsearch's
|
||||||
:meth:`~elasticsearch.Elasticsearch.bulk` api.
|
:meth:`~elasticsearch.Elasticsearch.bulk` api.
|
||||||
"""
|
"""
|
||||||
|
# when given a string, assume user wants to index raw json
|
||||||
|
if isinstance(data, six.string_types):
|
||||||
|
return '{"index": {}}', data
|
||||||
|
|
||||||
# make sure we don't alter the action
|
# make sure we don't alter the action
|
||||||
data = data.copy()
|
data = data.copy()
|
||||||
op_type = data.pop('_op_type', 'index')
|
op_type = data.pop('_op_type', 'index')
|
||||||
@@ -155,6 +161,11 @@ def streaming_bulk(client, actions, chunk_size=500, max_chunk_bytes=100 * 1014 *
|
|||||||
Alternatively, if `_source` is not present, it will pop all metadata fields
|
Alternatively, if `_source` is not present, it will pop all metadata fields
|
||||||
from the doc and use the rest as the document data.
|
from the doc and use the rest as the document data.
|
||||||
|
|
||||||
|
When reading raw json strings from a file, you can also pass them in. In
|
||||||
|
that case, however, you lose the ability to specify anything (index, type,
|
||||||
|
even id) on a per-record basis, all documents will just be sent to
|
||||||
|
elasticsearch to be indexed as-is.
|
||||||
|
|
||||||
The :meth:`~elasticsearch.Elasticsearch.bulk` api accepts `index`, `create`,
|
The :meth:`~elasticsearch.Elasticsearch.bulk` api accepts `index`, `create`,
|
||||||
`delete`, and `update` actions. Use the `_op_type` field to specify an
|
`delete`, and `update` actions. Use the `_op_type` field to specify an
|
||||||
action (`_op_type` defaults to `index`)::
|
action (`_op_type` defaults to `index`)::
|
||||||
|
|||||||
Reference in New Issue
Block a user