diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml index be5ec5f5..a78b2fc4 100644 --- a/.ci/test-matrix.yml +++ b/.ci/test-matrix.yml @@ -1,28 +1,14 @@ --- ELASTICSEARCH_VERSION: -- 6.4.2 -- 6.3.2 -- 6.1.4 -- 6.0.1 - +- 7.0.0-beta1 +- 6.6.2 PYTHON_VERSION: -- 2.6 - 2.7 -- 3.3 - 3.4 - 3.5 - 3.6 - 3.7 exclude: -# since a base image for python2.6 doens't exist i need to create one. -- PYTHON_VERSION: 2.6 -# Something is wrong with the way 3.7 and the es-client handle multiprocessing. -# https://github.com/elastic/elasticsearch-py/issues/865 -- PYTHON_VERSION: 3.7 -# new feature in 6.4.2 which is causing a test failure. -# https://github.com/elastic/elasticsearch-py/issues/866 -- ELASTICSEARCH_VERSION: 6.4.2 - diff --git a/docker-compose.yml b/docker-compose.yml index 1269da89..5cd32f1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,8 @@ services: - "repositories.url.allowed_urls=http://*" - node.attr.testattr=test - bootstrap.memory_lock=false + - "node.name=test" + - "cluster.initial_master_nodes=test" - "discovery.zen.ping.unicast.hosts=elasticsearch" - "http.max_content_length=5mb" networks: diff --git a/elasticsearch/client/__init__.py b/elasticsearch/client/__init__.py index 1f493f5c..afdbb1fe 100644 --- a/elasticsearch/client/__init__.py +++ b/elasticsearch/client/__init__.py @@ -15,7 +15,8 @@ from .tasks import TasksClient from .xpack import XPackClient from .utils import query_params, _make_path, SKIP_IN_PATH -logger = logging.getLogger('elasticsearch') +logger = logging.getLogger("elasticsearch") + def _normalize_hosts(hosts): """ @@ -34,7 +35,7 @@ def _normalize_hosts(hosts): # normalize hosts to dicts for host in hosts: if isinstance(host, string_types): - if '://' not in host: + if "://" not in host: host = "//%s" % host parsed_url = urlparse(host) @@ -44,15 +45,17 @@ def _normalize_hosts(hosts): h["port"] = parsed_url.port if parsed_url.scheme == "https": - h['port'] = parsed_url.port or 443 - h['use_ssl'] = True + h["port"] = parsed_url.port or 443 + h["use_ssl"] = True if parsed_url.username or parsed_url.password: - h['http_auth'] = '%s:%s' % (unquote(parsed_url.username), - unquote(parsed_url.password)) + h["http_auth"] = "%s:%s" % ( + unquote(parsed_url.username), + unquote(parsed_url.password), + ) - if parsed_url.path and parsed_url.path != '/': - h['url_prefix'] = parsed_url.path + if parsed_url.path and parsed_url.path != "/": + h["url_prefix"] = parsed_url.path out.append(h) else: @@ -184,6 +187,7 @@ class Elasticsearch(object): es = Elasticsearch(serializer=SetEncoder()) """ + def __init__(self, hosts=None, transport_class=Transport, **kwargs): """ :arg hosts: list of nodes we should connect to. Node should be a @@ -218,8 +222,8 @@ class Elasticsearch(object): cons = self.transport.hosts # truncate to 5 if there are too many if len(cons) > 5: - cons = cons[:5] + ['...'] - return '<{cls}({cons})>'.format(cls=self.__class__.__name__, cons=cons) + cons = cons[:5] + ["..."] + return "<{cls}({cons})>".format(cls=self.__class__.__name__, cons=cons) except: # probably operating on custom transport and connection_pool, ignore return super(Elasticsearch, self).__repr__() @@ -227,11 +231,11 @@ class Elasticsearch(object): def _bulk_body(self, body): # if not passed in a string, serialize items and join by newline if not isinstance(body, string_types): - body = '\n'.join(map(self.transport.serializer.dumps, body)) + body = "\n".join(map(self.transport.serializer.dumps, body)) # bulk body must end with a newline - if not body.endswith('\n'): - body += '\n' + if not body.endswith("\n"): + body += "\n" return body @@ -242,7 +246,7 @@ class Elasticsearch(object): ``_ """ try: - return self.transport.perform_request('HEAD', '/', params=params) + return self.transport.perform_request("HEAD", "/", params=params) except TransportError: return False @@ -252,18 +256,27 @@ class Elasticsearch(object): Get the basic info from the current cluster. ``_ """ - return self.transport.perform_request('GET', '/', params=params) + return self.transport.perform_request("GET", "/", params=params) - @query_params('parent', 'pipeline', 'refresh', 'routing', 'timeout', - 'timestamp', 'ttl', 'version', 'version_type', 'wait_for_active_shards') - def create(self, index, doc_type, id, body, params=None): + @query_params( + "parent", + "pipeline", + "refresh", + "routing", + "timeout", + "timestamp", + "ttl", + "version", + "version_type", + "wait_for_active_shards", + ) + def create(self, index, id, body, doc_type="_doc", params=None): """ Adds a typed JSON document in a specific index, making it searchable. Behind the scenes this method calls index(..., op_type='create') ``_ :arg index: The name of the index - :arg doc_type: The type of the document :arg id: Document ID :arg body: The document :arg parent: ID of the parent document @@ -286,24 +299,37 @@ class Elasticsearch(object): otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) """ - for param in (index, doc_type, id, body): + for param in (index, id, body): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path(index, doc_type, - id, '_create'), params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path(index, doc_type, id, "_create"), params=params, body=body + ) - @query_params('op_type', 'parent', 'pipeline', 'refresh', 'routing', - 'timeout', 'timestamp', 'ttl', 'version', 'version_type', - 'wait_for_active_shards') - def index(self, index, doc_type, body, id=None, params=None): + @query_params( + "if_seq_no", + "if_primary_term", + "op_type", + "parent", + "pipeline", + "refresh", + "routing", + "timeout", + "timestamp", + "ttl", + "version", + "version_type", + "wait_for_active_shards", + ) + def index(self, index, body, doc_type="_doc", id=None, params=None): """ Adds or updates a typed JSON document in a specific index, making it searchable. ``_ :arg index: The name of the index - :arg doc_type: The type of the document :arg body: The document :arg id: Document ID + :arg doc_type: Document type, defaults to `_doc`. Not used on ES 7 clusters. :arg op_type: Explicit operation type, default 'index', valid choices are: 'index', 'create' :arg parent: ID of the parent document @@ -326,23 +352,37 @@ class Elasticsearch(object): otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) """ - for param in (index, doc_type, body): + for param in (index, body): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST' if id in SKIP_IN_PATH else 'PUT', - _make_path(index, doc_type, id), params=params, body=body) + return self.transport.perform_request( + "POST" if id in SKIP_IN_PATH else "PUT", + _make_path(index, doc_type, id), + params=params, + body=body, + ) - @query_params('_source', '_source_exclude', '_source_include', 'parent', - 'preference', 'realtime', 'refresh', 'routing', 'stored_fields', - 'version', 'version_type') - def exists(self, index, doc_type, id, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "parent", + "preference", + "realtime", + "refresh", + "routing", + "stored_fields", + "version", + "version_type", + ) + def exists(self, index, id, doc_type="_doc", params=None): """ Returns a boolean indicating whether or not given document exists in Elasticsearch. ``_ :arg index: The name of the index - :arg doc_type: The type of the document (use `_all` to fetch the first - document matching the ID across all types) :arg id: The document ID :arg _source: True or false to return the _source field or not, or a list of fields to return @@ -364,22 +404,32 @@ class Elasticsearch(object): :arg version_type: Specific version type, valid choices are: 'internal', 'external', 'external_gte', 'force' """ - for param in (index, doc_type, id): + for param in (index, id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('HEAD', _make_path(index, - doc_type, id), params=params) + return self.transport.perform_request( + "HEAD", _make_path(index, doc_type, id), params=params + ) - @query_params('_source', '_source_exclude', '_source_include', 'parent', - 'preference', 'realtime', 'refresh', 'routing', 'version', - 'version_type') - def exists_source(self, index, doc_type, id, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "parent", + "preference", + "realtime", + "refresh", + "routing", + "version", + "version_type", + ) + def exists_source(self, index, id, doc_type=None, params=None): """ ``_ :arg index: The name of the index - :arg doc_type: The type of the document; use `_all` to fetch the first - document matching the ID across all types :arg id: The document ID :arg _source: True or false to return the _source field or not, or a list of fields to return @@ -399,23 +449,34 @@ class Elasticsearch(object): :arg version_type: Specific version type, valid choices are: 'internal', 'external', 'external_gte', 'force' """ - for param in (index, doc_type, id): + for param in (index, id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('HEAD', _make_path(index, - doc_type, id, '_source'), params=params) + return self.transport.perform_request( + "HEAD", _make_path(index, doc_type, id, "_source"), params=params + ) - @query_params('_source', '_source_exclude', '_source_include', 'parent', - 'preference', 'realtime', 'refresh', 'routing', 'stored_fields', - 'version', 'version_type') - def get(self, index, doc_type, id, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "parent", + "preference", + "realtime", + "refresh", + "routing", + "stored_fields", + "version", + "version_type", + ) + def get(self, index, id, doc_type="_doc", params=None): """ Get a typed JSON document from the index based on its id. ``_ :arg index: The name of the index - :arg doc_type: The type of the document (use `_all` to fetch the first - document matching the ID across all types) :arg id: The document ID :arg _source: True or false to return the _source field or not, or a list of fields to return @@ -437,23 +498,33 @@ class Elasticsearch(object): :arg version_type: Specific version type, valid choices are: 'internal', 'external', 'external_gte', 'force' """ - for param in (index, doc_type, id): + for param in (index, id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('GET', _make_path(index, - doc_type, id), params=params) + return self.transport.perform_request( + "GET", _make_path(index, doc_type, id), params=params + ) - @query_params('_source', '_source_exclude', '_source_include', 'parent', - 'preference', 'realtime', 'refresh', 'routing', 'version', - 'version_type') - def get_source(self, index, doc_type, id, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "parent", + "preference", + "realtime", + "refresh", + "routing", + "version", + "version_type", + ) + def get_source(self, index, id, doc_type="_doc", params=None): """ Get the source of a document by it's index, type and id. ``_ :arg index: The name of the index - :arg doc_type: The type of the document; use `_all` to fetch the first - document matching the ID across all types :arg id: The document ID :arg _source: True or false to return the _source field or not, or a list of fields to return @@ -473,15 +544,26 @@ class Elasticsearch(object): :arg version_type: Specific version type, valid choices are: 'internal', 'external', 'external_gte', 'force' """ - for param in (index, doc_type, id): + for param in (index, id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('GET', _make_path(index, - doc_type, id, '_source'), params=params) + return self.transport.perform_request( + "GET", _make_path(index, doc_type, id, "_source"), params=params + ) - @query_params('_source', '_source_exclude', '_source_include', 'preference', - 'realtime', 'refresh', 'routing', 'stored_fields') - def mget(self, body, index=None, doc_type=None, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "preference", + "realtime", + "refresh", + "routing", + "stored_fields", + ) + def mget(self, body, doc_type=None, index=None, params=None): """ Get multiple documents based on an index, type (optional) and ids. ``_ @@ -490,7 +572,6 @@ class Elasticsearch(object): document information) or `ids` (when index and type is provided in the URL. :arg index: The name of the index - :arg doc_type: The type of the document :arg _source: True or false to return the _source field or not, or a list of fields to return :arg _source_exclude: A list of fields to exclude from the returned @@ -509,19 +590,37 @@ class Elasticsearch(object): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_mget'), params=params, body=body) + return self.transport.perform_request( + "GET", _make_path(index, doc_type, "_mget"), params=params, body=body + ) - @query_params('_source', '_source_exclude', '_source_include', 'fields', - 'lang', 'parent', 'refresh', 'retry_on_conflict', 'routing', 'timeout', - 'timestamp', 'ttl', 'version', 'version_type', 'wait_for_active_shards') - def update(self, index, doc_type, id, body=None, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "fields", + "if_seq_no", + "if_primary_term", + "lang", + "parent", + "refresh", + "retry_on_conflict", + "routing", + "timeout", + "timestamp", + "ttl", + "version", + "version_type", + "wait_for_active_shards", + ) + def update(self, index, id, doc_type="_doc", body=None, params=None): """ Update a document based on a script or partial data provided. ``_ :arg index: The name of the index - :arg doc_type: The type of the document :arg id: Document ID :arg body: The request definition using either `script` or partial `doc` :arg _source: True or false to return the _source field or not, or a @@ -531,6 +630,8 @@ class Elasticsearch(object): :arg _source_include: A list of fields to extract and return from the _source field :arg fields: A comma-separated list of fields to return in the response + :arg if_seq_no: + :arg if_primary_term: :arg lang: The script language (default: painless) :arg parent: ID of the parent document. Is is only used for routing and when for the upsert request @@ -538,8 +639,7 @@ class Elasticsearch(object): operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes., valid choices are: 'true', 'false', - 'wait_for' - :arg retry_on_conflict: Specify how many times should the operation be + 'wait_forarg retry_on_conflict: Specify how many times should the operation be retried when a conflict occurs (default: 0) :arg routing: Specific routing value :arg timeout: Explicit operation timeout @@ -554,31 +654,64 @@ class Elasticsearch(object): copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) """ - for param in (index, doc_type, id): + for param in (index, id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST', _make_path(index, - doc_type, id, '_update'), params=params, body=body) + return self.transport.perform_request( + "POST", _make_path(index, doc_type, id, "_update"), params=params, body=body + ) - @query_params('_source', '_source_exclude', '_source_include', - 'allow_no_indices', 'allow_partial_search_results', 'analyze_wildcard', - 'analyzer', 'batched_reduce_size', 'default_operator', 'df', - 'docvalue_fields', 'expand_wildcards', 'explain', 'from_', - 'ignore_unavailable', 'lenient', 'max_concurrent_shard_requests', - 'pre_filter_shard_size', 'preference', 'q', 'request_cache', 'routing', - 'scroll', 'search_type', 'size', 'sort', 'stats', 'stored_fields', - 'suggest_field', 'suggest_mode', 'suggest_size', 'suggest_text', - 'terminate_after', 'timeout', 'track_scores', 'track_total_hits', - 'typed_keys', 'version') - def search(self, index=None, doc_type=None, body=None, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "allow_no_indices", + "allow_partial_search_results", + "analyze_wildcard", + "analyzer", + "batched_reduce_size", + "default_operator", + "df", + "docvalue_fields", + "expand_wildcards", + "explain", + "from_", + "ignore_unavailable", + "lenient", + "max_concurrent_shard_requests", + "pre_filter_shard_size", + "preference", + "q", + "rest_total_hits_as_int", + "request_cache", + "routing", + "scroll", + "search_type", + "seq_no_primary_term", + "size", + "sort", + "stats", + "stored_fields", + "suggest_field", + "suggest_mode", + "suggest_size", + "suggest_text", + "terminate_after", + "timeout", + "track_scores", + "track_total_hits", + "typed_keys", + "version", + ) + def search(self, index=None, body=None, params=None): """ Execute a search query and get back search hits that match the query. ``_ :arg index: A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices - :arg doc_type: A comma-separated list of document types to search; leave - empty to perform the operation on all types :arg body: The search definition using the Query DSL :arg _source: True or false to return the _source field or not, or a list of fields to return @@ -632,6 +765,9 @@ class Elasticsearch(object): :arg preference: Specify the node or shard the operation should be performed on (default: random) :arg q: Query in the Lucene query string syntax + :arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number + in the response. This param is added version 6.x to handle mixed cluster queries where nodes + are in multiple versions (7.0 and 6.latest) :arg request_cache: Specify if request cache should be used for this request or not, defaults to index level setting :arg routing: A comma-separated list of specific routing values @@ -665,31 +801,60 @@ class Elasticsearch(object): hit """ # from is a reserved word so it cannot be used, use from_ instead - if 'from_' in params: - params['from'] = params.pop('from_') + if "from_" in params: + params["from"] = params.pop("from_") - if doc_type and not index: - index = '_all' - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_search'), params=params, body=body) + if not index: + index = "_all" + return self.transport.perform_request( + "GET", _make_path(index, "_search"), params=params, body=body + ) - @query_params('_source', '_source_exclude', '_source_include', - 'allow_no_indices', 'analyze_wildcard', 'analyzer', 'conflicts', - 'default_operator', 'df', 'expand_wildcards', 'from_', - 'ignore_unavailable', 'lenient', 'pipeline', 'preference', 'q', - 'refresh', 'request_cache', 'requests_per_second', 'routing', 'scroll', - 'scroll_size', 'search_timeout', 'search_type', 'size', 'slices', - 'sort', 'stats', 'terminate_after', 'timeout', 'version', - 'version_type', 'wait_for_active_shards', 'wait_for_completion') - def update_by_query(self, index, doc_type=None, body=None, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "allow_no_indices", + "analyze_wildcard", + "analyzer", + "conflicts", + "default_operator", + "df", + "expand_wildcards", + "from_", + "ignore_unavailable", + "lenient", + "pipeline", + "preference", + "q", + "refresh", + "request_cache", + "requests_per_second", + "routing", + "scroll", + "scroll_size", + "search_timeout", + "search_type", + "size", + "slices", + "sort", + "stats", + "terminate_after", + "timeout", + "version", + "version_type", + "wait_for_active_shards", + "wait_for_completion", + ) + def update_by_query(self, index, body=None, params=None): """ Perform an update on all documents matching a query. ``_ :arg index: A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices - :arg doc_type: A comma-separated list of document types to search; leave - empty to perform the operation on all types :arg body: The search definition using the Query DSL :arg _source: True or false to return the _source field or not, or a list of fields to return @@ -763,11 +928,18 @@ class Elasticsearch(object): """ if index in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('POST', _make_path(index, - doc_type, '_update_by_query'), params=params, body=body) + return self.transport.perform_request( + "POST", _make_path(index, "_update_by_query"), params=params, body=body + ) - @query_params('refresh', 'requests_per_second', 'slices', 'timeout', - 'wait_for_active_shards', 'wait_for_completion') + @query_params( + "refresh", + "requests_per_second", + "slices", + "timeout", + "wait_for_active_shards", + "wait_for_completion", + ) def reindex(self, body, params=None): """ Reindex all documents from one index to another. @@ -793,10 +965,11 @@ class Elasticsearch(object): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('POST', '/_reindex', - params=params, body=body) + return self.transport.perform_request( + "POST", "/_reindex", params=params, body=body + ) - @query_params('requests_per_second') + @query_params("requests_per_second") def reindex_rethrottle(self, task_id=None, params=None): """ Change the value of ``requests_per_second`` of a running ``reindex`` task. @@ -806,18 +979,47 @@ class Elasticsearch(object): :arg requests_per_second: The throttle to set on this request in floating sub-requests per second. -1 means set no throttle. """ - return self.transport.perform_request('POST', _make_path('_reindex', - task_id, '_rethrottle'), params=params) + return self.transport.perform_request( + "POST", _make_path("_reindex", task_id, "_rethrottle"), params=params + ) - @query_params('_source', '_source_exclude', '_source_include', - 'allow_no_indices', 'analyze_wildcard', 'analyzer', 'conflicts', - 'default_operator', 'df', 'expand_wildcards', 'from_', - 'ignore_unavailable', 'lenient', 'preference', 'q', 'refresh', - 'request_cache', 'requests_per_second', 'routing', 'scroll', - 'scroll_size', 'search_timeout', 'search_type', 'size', 'slices', - 'sort', 'stats', 'terminate_after', 'timeout', 'version', - 'wait_for_active_shards', 'wait_for_completion') - def delete_by_query(self, index, body, doc_type=None, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "allow_no_indices", + "analyze_wildcard", + "analyzer", + "conflicts", + "default_operator", + "df", + "expand_wildcards", + "from_", + "ignore_unavailable", + "lenient", + "preference", + "q", + "refresh", + "request_cache", + "requests_per_second", + "routing", + "scroll", + "scroll_size", + "search_timeout", + "search_type", + "size", + "slices", + "sort", + "stats", + "terminate_after", + "timeout", + "version", + "wait_for_active_shards", + "wait_for_completion", + ) + def delete_by_query(self, index, body, params=None): """ Delete all documents matching a query. ``_ @@ -825,8 +1027,6 @@ class Elasticsearch(object): :arg index: A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices :arg body: The search definition using the Query DSL - :arg doc_type: A comma-separated list of document types to search; leave - empty to perform the operation on all types :arg _source: True or false to return the _source field or not, or a list of fields to return :arg _source_exclude: A list of fields to exclude from the returned @@ -896,12 +1096,19 @@ class Elasticsearch(object): for param in (index, body): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST', _make_path(index, - doc_type, '_delete_by_query'), params=params, body=body) + return self.transport.perform_request( + "POST", _make_path(index, "_delete_by_query"), params=params, body=body + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'local', 'preference', 'routing') - def search_shards(self, index=None, doc_type=None, params=None): + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "local", + "preference", + "routing", + ) + def search_shards(self, index=None, params=None): """ The search shards api returns the indices and shards that a search request would be executed against. This can give useful feedback for working @@ -924,13 +1131,23 @@ class Elasticsearch(object): performed on (default: random) :arg routing: Specific routing value """ - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_search_shards'), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_search_shards"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'explain', - 'ignore_unavailable', 'preference', 'profile', 'routing', 'scroll', - 'search_type', 'typed_keys') - def search_template(self, index=None, doc_type=None, body=None, params=None): + @query_params( + "allow_no_indices", + "expand_wildcards", + "explain", + "ignore_unavailable", + "preference", + "profile", + "routing", + "scroll", + "search_type", + "typed_keys", + ) + def search_template(self, index=None, body=None, params=None): """ A query that accepts a query template and a map of key/value pairs to fill in template parameters. @@ -938,8 +1155,6 @@ class Elasticsearch(object): :arg index: A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices - :arg doc_type: A comma-separated list of document types to search; leave - empty to perform the operation on all types :arg body: The search definition template and its params :arg allow_no_indices: Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` @@ -963,13 +1178,28 @@ class Elasticsearch(object): :arg typed_keys: Specify whether aggregation and suggester names should be prefixed by their respective types in the response """ - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_search', 'template'), params=params, body=body) + return self.transport.perform_request( + "GET", _make_path(index, "_search", "template"), params=params, body=body + ) - @query_params('_source', '_source_exclude', '_source_include', - 'analyze_wildcard', 'analyzer', 'default_operator', 'df', 'lenient', - 'parent', 'preference', 'q', 'routing', 'stored_fields') - def explain(self, index, doc_type, id, body=None, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "analyze_wildcard", + "analyzer", + "default_operator", + "df", + "lenient", + "parent", + "preference", + "q", + "routing", + "stored_fields", + ) + def explain(self, index, id, doc_type="_doc", body=None, params=None): """ The explain api computes a score explanation for a query and a specific document. This can give useful feedback whether a document matches or @@ -977,7 +1207,6 @@ class Elasticsearch(object): ``_ :arg index: The name of the index - :arg doc_type: The type of the document :arg id: The document ID :arg body: The query definition using the Query DSL :arg _source: True or false to return the _source field or not, or a @@ -1002,13 +1231,14 @@ class Elasticsearch(object): :arg stored_fields: A comma-separated list of stored fields to return in the response """ - for param in (index, doc_type, id): + for param in (index, id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('GET', _make_path(index, - doc_type, id, '_explain'), params=params, body=body) + return self.transport.perform_request( + "GET", _make_path(index, doc_type, id, "_explain"), params=params, body=body + ) - @query_params('scroll') + @query_params("scroll", "rest_total_hits_as_int") def scroll(self, scroll_id=None, body=None, params=None): """ Scroll a search request created by specifying the scroll parameter. @@ -1018,16 +1248,20 @@ class Elasticsearch(object): :arg body: The scroll ID if not passed by URL or query parameter. :arg scroll: Specify how long a consistent view of the index should be maintained for scrolled search + :arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number + in the response. This param is added version 6.x to handle mixed cluster queries where nodes + are in multiple versions (7.0 and 6.latest) """ if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: raise ValueError("You need to supply scroll_id or body.") elif scroll_id and not body: - body = {'scroll_id':scroll_id} + body = {"scroll_id": scroll_id} elif scroll_id: - params['scroll_id'] = scroll_id + params["scroll_id"] = scroll_id - return self.transport.perform_request('GET', '/_search/scroll', - params=params, body=body) + return self.transport.perform_request( + "GET", "/_search/scroll", params=params, body=body + ) @query_params() def clear_scroll(self, scroll_id=None, body=None, params=None): @@ -1043,22 +1277,31 @@ class Elasticsearch(object): if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: raise ValueError("You need to supply scroll_id or body.") elif scroll_id and not body: - body = {'scroll_id':[scroll_id]} + body = {"scroll_id": [scroll_id]} elif scroll_id: - params['scroll_id'] = scroll_id + params["scroll_id"] = scroll_id - return self.transport.perform_request('DELETE', '/_search/scroll', - params=params, body=body) + return self.transport.perform_request( + "DELETE", "/_search/scroll", params=params, body=body + ) - @query_params('parent', 'refresh', 'routing', 'timeout', 'version', - 'version_type', 'wait_for_active_shards') - def delete(self, index, doc_type, id, params=None): + @query_params( + "if_seq_no", + "if_primary_term", + "parent", + "refresh", + "routing", + "timeout", + "version", + "version_type", + "wait_for_active_shards", + ) + def delete(self, index, id, doc_type="_doc", params=None): """ Delete a typed JSON document from a specific index based on its id. ``_ :arg index: The name of the index - :arg doc_type: The type of the document :arg id: The document ID :arg parent: ID of parent document :arg refresh: If `true` then refresh the effected shards to make this @@ -1077,22 +1320,34 @@ class Elasticsearch(object): copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) """ - for param in (index, doc_type, id): + for param in (index, id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('DELETE', _make_path(index, - doc_type, id), params=params) + return self.transport.perform_request( + "DELETE", _make_path(index, doc_type, id), params=params + ) - @query_params('allow_no_indices', 'analyze_wildcard', 'analyzer', - 'default_operator', 'df', 'expand_wildcards', 'ignore_unavailable', - 'lenient', 'min_score', 'preference', 'q', 'routing', 'terminate_after') - def count(self, index=None, doc_type=None, body=None, params=None): + @query_params( + "allow_no_indices", + "analyze_wildcard", + "analyzer", + "default_operator", + "df", + "expand_wildcards", + "ignore_unavailable", + "lenient", + "min_score", + "preference", + "q", + "routing", + "terminate_after", + ) + def count(self, doc_type=None, index=None, body=None, params=None): """ Execute a query and get the number of matches for that query. ``_ :arg index: A comma-separated list of indices to restrict the results - :arg doc_type: A comma-separated list of types to restrict the results :arg body: A query to restrict the results specified with the Query DSL (optional) :arg allow_no_indices: Whether to ignore if a wildcard indices @@ -1119,15 +1374,27 @@ class Elasticsearch(object): :arg q: Query in the Lucene query string syntax :arg routing: Specific routing value """ - if doc_type and not index: - index = '_all' + if not index: + index = "_all" - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_count'), params=params, body=body) + return self.transport.perform_request( + "GET", _make_path(index, doc_type, "_count"), params=params, body=body + ) - @query_params('_source', '_source_exclude', '_source_include', 'fields', - 'pipeline', 'refresh', 'routing', 'timeout', 'wait_for_active_shards') - def bulk(self, body, index=None, doc_type=None, params=None): + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "fields", + "pipeline", + "refresh", + "routing", + "timeout", + "wait_for_active_shards", + ) + def bulk(self, body, doc_type=None, index=None, params=None): """ Perform many index/delete operations in a single API call. @@ -1138,7 +1405,6 @@ class Elasticsearch(object): :arg body: The operation definition and data (action-data pairs), separated by newlines :arg index: Default index for items which don't provide one - :arg doc_type: Default document type for items which don't provide one :arg _source: True or false to return the _source field or not, or default list of fields to return, can be overridden on each sub- request @@ -1164,13 +1430,23 @@ class Elasticsearch(object): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('POST', _make_path(index, - doc_type, '_bulk'), params=params, body=self._bulk_body(body), - headers={'content-type': 'application/x-ndjson'}) + return self.transport.perform_request( + "POST", + _make_path(index, doc_type, "_bulk"), + params=params, + body=self._bulk_body(body), + headers={"content-type": "application/x-ndjson"}, + ) - @query_params('max_concurrent_searches', 'pre_filter_shard_size', - 'search_type', 'typed_keys') - def msearch(self, body, index=None, doc_type=None, params=None): + @query_params( + "max_concurrent_searches", + "max_concurrent_shard_requests", + "pre_filter_shard_size", + "rest_total_hits_as_int", + "search_type", + "typed_keys", + ) + def msearch(self, body, index=None, params=None): """ Execute several search requests within the same API. ``_ @@ -1178,8 +1454,6 @@ class Elasticsearch(object): :arg body: The request definitions (metadata-search request definition pairs), separated by newlines :arg index: A comma-separated list of index names to use as default - :arg doc_type: A comma-separated list of document types to use as - default :arg max_concurrent_searches: Controls the maximum number of concurrent searches the multi search api will execute :arg pre_filter_shard_size: A threshold that enforces a pre-filter @@ -1197,14 +1471,29 @@ class Elasticsearch(object): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_msearch'), params=params, body=self._bulk_body(body), - headers={'content-type': 'application/x-ndjson'}) + return self.transport.perform_request( + "GET", + _make_path(index, "_msearch"), + params=params, + body=self._bulk_body(body), + headers={"content-type": "application/x-ndjson"}, + ) - @query_params('field_statistics', 'fields', 'offsets', 'parent', 'payloads', - 'positions', 'preference', 'realtime', 'routing', 'term_statistics', - 'version', 'version_type') - def termvectors(self, index, doc_type, id=None, body=None, params=None): + @query_params( + "field_statistics", + "fields", + "offsets", + "parent", + "payloads", + "positions", + "preference", + "realtime", + "routing", + "term_statistics", + "version", + "version_type", + ) + def termvectors(self, index, doc_type="_doc", id=None, body=None, params=None): """ Returns information and statistics on terms in the fields of a particular document. The document could be stored in the index or @@ -1214,7 +1503,6 @@ class Elasticsearch(object): ``_ :arg index: The index in which the document resides. - :arg doc_type: The type of the document. :arg id: The id of the document, when not specified a doc param should be supplied. :arg body: Define parameters and or supply a document to get termvectors @@ -1241,23 +1529,38 @@ class Elasticsearch(object): :arg version_type: Specific version type, valid choices are: 'internal', 'external', 'external_gte', 'force' """ - for param in (index, doc_type): + for param in (index,): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('GET', _make_path(index, - doc_type, id, '_termvectors'), params=params, body=body) + return self.transport.perform_request( + "GET", + _make_path(index, doc_type, id, "_termvectors"), + params=params, + body=body, + ) - @query_params('field_statistics', 'fields', 'ids', 'offsets', 'parent', - 'payloads', 'positions', 'preference', 'realtime', 'routing', - 'term_statistics', 'version', 'version_type') - def mtermvectors(self, index=None, doc_type=None, body=None, params=None): + @query_params( + "field_statistics", + "fields", + "ids", + "offsets", + "parent", + "payloads", + "positions", + "preference", + "realtime", + "routing", + "term_statistics", + "version", + "version_type", + ) + def mtermvectors(self, doc_type=None, index=None, body=None, params=None): """ Multi termvectors API allows to get multiple termvectors based on an index, type and id. ``_ :arg index: The index in which the document resides. - :arg doc_type: The type of the document. :arg body: Define ids, documents, parameters or a list of parameters per document here. You must at least provide a list of document ids. See documentation. @@ -1296,8 +1599,12 @@ class Elasticsearch(object): :arg version_type: Specific version type, valid choices are: 'internal', 'external', 'external_gte', 'force' """ - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_mtermvectors'), params=params, body=body) + return self.transport.perform_request( + "GET", + _make_path(index, doc_type, "_mtermvectors"), + params=params, + body=body, + ) @query_params() def put_script(self, id, body, context=None, params=None): @@ -1311,8 +1618,9 @@ class Elasticsearch(object): for param in (id, body): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_scripts', id, - context), params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path("_scripts", id, context), params=params, body=body + ) @query_params() def get_script(self, id, params=None): @@ -1324,8 +1632,9 @@ class Elasticsearch(object): """ if id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'id'.") - return self.transport.perform_request('GET', _make_path('_scripts', id), - params=params) + return self.transport.perform_request( + "GET", _make_path("_scripts", id), params=params + ) @query_params() def delete_script(self, id, params=None): @@ -1337,8 +1646,9 @@ class Elasticsearch(object): """ if id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'id'.") - return self.transport.perform_request('DELETE', _make_path('_scripts', - id), params=params) + return self.transport.perform_request( + "DELETE", _make_path("_scripts", id), params=params + ) @query_params() def render_search_template(self, id=None, body=None, params=None): @@ -1348,11 +1658,12 @@ class Elasticsearch(object): :arg id: The id of the stored search template :arg body: The search definition template and its params """ - return self.transport.perform_request('GET', _make_path('_render', - 'template', id), params=params, body=body) + return self.transport.perform_request( + "GET", _make_path("_render", "template", id), params=params, body=body + ) - @query_params('max_concurrent_searches', 'search_type', 'typed_keys') - def msearch_template(self, body, index=None, doc_type=None, params=None): + @query_params("max_concurrent_searches", "search_type", "typed_keys") + def msearch_template(self, body, index=None, params=None): """ The /_search/template endpoint allows to use the mustache language to pre render search requests, before they are executed and fill existing @@ -1362,8 +1673,6 @@ class Elasticsearch(object): :arg body: The request definitions (metadata-search request definition pairs), separated by newlines :arg index: A comma-separated list of index names to use as default - :arg doc_type: A comma-separated list of document types to use as - default :arg max_concurrent_searches: Controls the maximum number of concurrent searches the multi search api will execute :arg search_type: Search operation type, valid choices are: @@ -1374,12 +1683,17 @@ class Elasticsearch(object): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('GET', _make_path(index, doc_type, - '_msearch', 'template'), params=params, body=self._bulk_body(body), - headers={'content-type': 'application/x-ndjson'}) + return self.transport.perform_request( + "GET", + _make_path(index, "_msearch", "template"), + params=params, + body=self._bulk_body(body), + headers={"content-type": "application/x-ndjson"}, + ) - @query_params('allow_no_indices', 'expand_wildcards', 'fields', - 'ignore_unavailable') + @query_params( + "allow_no_indices", "expand_wildcards", "fields", "ignore_unavailable" + ) def field_caps(self, index=None, body=None, params=None): """ The field capabilities API allows to retrieve the capabilities of fields among multiple indices. @@ -1398,6 +1712,6 @@ class Elasticsearch(object): :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed) """ - return self.transport.perform_request('GET', _make_path(index, - '_field_caps'), params=params, body=body) - + return self.transport.perform_request( + "GET", _make_path(index, "_field_caps"), params=params, body=body + ) diff --git a/elasticsearch/client/indices.py b/elasticsearch/client/indices.py index a4204306..6667c8bd 100644 --- a/elasticsearch/client/indices.py +++ b/elasticsearch/client/indices.py @@ -1,7 +1,8 @@ from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class IndicesClient(NamespacedClient): - @query_params('format', 'prefer_local') + @query_params("format", "prefer_local") def analyze(self, index=None, body=None, params=None): """ Perform the analysis process on a text and return the tokens breakdown of the text. @@ -15,10 +16,11 @@ class IndicesClient(NamespacedClient): :arg prefer_local: With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true) """ - return self.transport.perform_request('GET', _make_path(index, - '_analyze'), params=params, body=body) + return self.transport.perform_request( + "GET", _make_path(index, "_analyze"), params=params, body=body + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable') + @query_params("allow_no_indices", "expand_wildcards", "ignore_unavailable") def refresh(self, index=None, params=None): """ Explicitly refresh one or more index, making all operations performed @@ -36,11 +38,17 @@ class IndicesClient(NamespacedClient): :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed) """ - return self.transport.perform_request('POST', _make_path(index, - '_refresh'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_refresh"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'force', - 'ignore_unavailable', 'wait_if_ongoing') + @query_params( + "allow_no_indices", + "expand_wildcards", + "force", + "ignore_unavailable", + "wait_if_ongoing", + ) def flush(self, index=None, params=None): """ Explicitly flush one or more indices. @@ -66,10 +74,13 @@ class IndicesClient(NamespacedClient): already executing. The default is true. If set to false the flush will be skipped iff if another flush operation is already running. """ - return self.transport.perform_request('POST', _make_path(index, - '_flush'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_flush"), params=params + ) - @query_params('master_timeout', 'timeout', 'wait_for_active_shards', 'include_type_name') + @query_params( + "master_timeout", "timeout", "wait_for_active_shards", "include_type_name" + ) def create(self, index, body=None, params=None): """ Create an index in Elasticsearch. @@ -86,11 +97,19 @@ class IndicesClient(NamespacedClient): """ if index in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('PUT', _make_path(index), - params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path(index), params=params, body=body + ) - @query_params('allow_no_indices', 'expand_wildcards', 'flat_settings', - 'ignore_unavailable', 'include_defaults', 'local', 'include_type_name') + @query_params( + "allow_no_indices", + "expand_wildcards", + "flat_settings", + "ignore_unavailable", + "include_defaults", + "local", + "include_type_name", + ) def get(self, index, feature=None, params=None): """ The get index API allows to retrieve information about one or more indexes. @@ -113,11 +132,17 @@ class IndicesClient(NamespacedClient): """ if index in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('GET', _make_path(index, - feature), params=params) + return self.transport.perform_request( + "GET", _make_path(index, feature), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'master_timeout', 'timeout') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "master_timeout", + "timeout", + ) def open(self, index, params=None): """ Open a closed index to make it available for search. @@ -137,11 +162,17 @@ class IndicesClient(NamespacedClient): """ if index in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('POST', _make_path(index, - '_open'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_open"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'master_timeout', 'timeout') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "master_timeout", + "timeout", + ) def close(self, index, params=None): """ Close an index to remove it's overhead from the cluster. Closed index @@ -162,11 +193,17 @@ class IndicesClient(NamespacedClient): """ if index in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('POST', _make_path(index, - '_close'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_close"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'master_timeout', 'timeout') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "master_timeout", + "timeout", + ) def delete(self, index, params=None): """ Delete an index in Elasticsearch @@ -185,11 +222,18 @@ class IndicesClient(NamespacedClient): """ if index in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('DELETE', _make_path(index), - params=params) + return self.transport.perform_request( + "DELETE", _make_path(index), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'flat_settings', - 'ignore_unavailable', 'include_defaults', 'local') + @query_params( + "allow_no_indices", + "expand_wildcards", + "flat_settings", + "ignore_unavailable", + "include_defaults", + "local", + ) def exists(self, index, params=None): """ Return a boolean indicating whether given index exists. @@ -210,11 +254,9 @@ class IndicesClient(NamespacedClient): """ if index in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('HEAD', _make_path(index), - params=params) + return self.transport.perform_request("HEAD", _make_path(index), params=params) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'local') + @query_params("allow_no_indices", "expand_wildcards", "ignore_unavailable", "local") def exists_type(self, index, doc_type, params=None): """ Check if a type/types exists in an index/indices. @@ -237,12 +279,19 @@ class IndicesClient(NamespacedClient): for param in (index, doc_type): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('HEAD', _make_path(index, - '_mapping', doc_type), params=params) + return self.transport.perform_request( + "HEAD", _make_path(index, "_mapping", doc_type), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'master_timeout', 'timeout','include_type_name') - def put_mapping(self, doc_type, body, index=None, params=None): + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "master_timeout", + "timeout", + "include_type_name", + ) + def put_mapping(self, body, doc_type=None, index=None, params=None): """ Register specific mapping definition for a specific type. ``_ @@ -265,14 +314,20 @@ class IndicesClient(NamespacedClient): :arg include_type_name: Specify whether requests and responses should include a type name (default: depends on Elasticsearch version). """ - for param in (doc_type, body): + for param in (body,): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path(index, - '_mapping', doc_type), params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path(index, "_mapping", doc_type), params=params, body=body + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'local','include_type_name') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "local", + "include_type_name", + ) def get_mapping(self, index=None, doc_type=None, params=None): """ Retrieve mapping definition of index or index/type. @@ -293,11 +348,18 @@ class IndicesClient(NamespacedClient): :arg include_type_name: Specify whether requests and responses should include a type name (default: depends on Elasticsearch version). """ - return self.transport.perform_request('GET', _make_path(index, - '_mapping', doc_type), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_mapping", doc_type), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'include_defaults', 'local', 'include_type_name') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "include_defaults", + "local", + "include_type_name", + ) def get_field_mapping(self, fields, index=None, doc_type=None, params=None): """ Retrieve mapping definition of a specific field. @@ -323,10 +385,13 @@ class IndicesClient(NamespacedClient): """ if fields in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'fields'.") - return self.transport.perform_request('GET', _make_path(index, - '_mapping', doc_type, 'field', fields), params=params) + return self.transport.perform_request( + "GET", + _make_path(index, "_mapping", doc_type, "field", fields), + params=params, + ) - @query_params('master_timeout', 'timeout') + @query_params("master_timeout", "timeout") def put_alias(self, index, name, body=None, params=None): """ Create an alias for a specific index/indices. @@ -343,11 +408,11 @@ class IndicesClient(NamespacedClient): for param in (index, name): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path(index, - '_alias', name), params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path(index, "_alias", name), params=params, body=body + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'local') + @query_params("allow_no_indices", "expand_wildcards", "ignore_unavailable", "local") def exists_alias(self, index=None, name=None, params=None): """ Return a boolean indicating whether given alias exists. @@ -366,11 +431,11 @@ class IndicesClient(NamespacedClient): :arg local: Return local information, do not retrieve the state from master node (default: false) """ - return self.transport.perform_request('HEAD', _make_path(index, '_alias', - name), params=params) + return self.transport.perform_request( + "HEAD", _make_path(index, "_alias", name), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'local') + @query_params("allow_no_indices", "expand_wildcards", "ignore_unavailable", "local") def get_alias(self, index=None, name=None, params=None): """ Retrieve a specified alias. @@ -389,10 +454,11 @@ class IndicesClient(NamespacedClient): :arg local: Return local information, do not retrieve the state from master node (default: false) """ - return self.transport.perform_request('GET', _make_path(index, - '_alias', name), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_alias", name), params=params + ) - @query_params('master_timeout', 'timeout') + @query_params("master_timeout", "timeout") def update_aliases(self, body, params=None): """ Update specified aliases. @@ -404,10 +470,11 @@ class IndicesClient(NamespacedClient): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('POST', '/_aliases', - params=params, body=body) + return self.transport.perform_request( + "POST", "/_aliases", params=params, body=body + ) - @query_params('master_timeout', 'timeout') + @query_params("master_timeout", "timeout") def delete_alias(self, index, name, params=None): """ Delete specific alias. @@ -424,11 +491,18 @@ class IndicesClient(NamespacedClient): for param in (index, name): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('DELETE', _make_path(index, - '_alias', name), params=params) + return self.transport.perform_request( + "DELETE", _make_path(index, "_alias", name), params=params + ) - @query_params('create', 'flat_settings', 'master_timeout', 'order', - 'timeout', 'include_type_name') + @query_params( + "create", + "flat_settings", + "master_timeout", + "order", + "timeout", + "include_type_name", + ) def put_template(self, name, body, params=None): """ Create an index template that will automatically be applied to new @@ -450,10 +524,11 @@ class IndicesClient(NamespacedClient): for param in (name, body): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_template', - name), params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path("_template", name), params=params, body=body + ) - @query_params('flat_settings', 'local', 'master_timeout') + @query_params("flat_settings", "local", "master_timeout") def exists_template(self, name, params=None): """ Return a boolean indicating whether given template exists. @@ -468,10 +543,11 @@ class IndicesClient(NamespacedClient): """ if name in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'name'.") - return self.transport.perform_request('HEAD', _make_path('_template', - name), params=params) + return self.transport.perform_request( + "HEAD", _make_path("_template", name), params=params + ) - @query_params('flat_settings', 'local', 'master_timeout', 'include_type_name') + @query_params("flat_settings", "local", "master_timeout", "include_type_name") def get_template(self, name=None, params=None): """ Retrieve an index template by its name. @@ -486,10 +562,11 @@ class IndicesClient(NamespacedClient): :arg include_type_name: Specify whether requests and responses should include a type name (default: depends on Elasticsearch version). """ - return self.transport.perform_request('GET', _make_path('_template', - name), params=params) + return self.transport.perform_request( + "GET", _make_path("_template", name), params=params + ) - @query_params('master_timeout', 'timeout') + @query_params("master_timeout", "timeout") def delete_template(self, name, params=None): """ Delete an index template by its name. @@ -501,11 +578,18 @@ class IndicesClient(NamespacedClient): """ if name in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'name'.") - return self.transport.perform_request('DELETE', - _make_path('_template', name), params=params) + return self.transport.perform_request( + "DELETE", _make_path("_template", name), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'flat_settings', - 'ignore_unavailable', 'include_defaults', 'local') + @query_params( + "allow_no_indices", + "expand_wildcards", + "flat_settings", + "ignore_unavailable", + "include_defaults", + "local", + ) def get_settings(self, index=None, name=None, params=None): """ Retrieve settings for one or more (or all) indices. @@ -528,11 +612,18 @@ class IndicesClient(NamespacedClient): :arg local: Return local information, do not retrieve the state from master node (default: false) """ - return self.transport.perform_request('GET', _make_path(index, - '_settings', name), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_settings", name), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'flat_settings', - 'ignore_unavailable', 'master_timeout', 'preserve_existing') + @query_params( + "allow_no_indices", + "expand_wildcards", + "flat_settings", + "ignore_unavailable", + "master_timeout", + "preserve_existing", + ) def put_settings(self, body, index=None, params=None): """ Change specific index level settings in real time. @@ -557,11 +648,19 @@ class IndicesClient(NamespacedClient): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('PUT', _make_path(index, - '_settings'), params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path(index, "_settings"), params=params, body=body + ) - @query_params('completion_fields', 'fielddata_fields', 'fields', 'groups', - 'include_segment_file_sizes', 'level', 'types') + @query_params( + "completion_fields", + "fielddata_fields", + "fields", + "groups", + "include_segment_file_sizes", + "level", + "types", + ) def stats(self, index=None, metric=None, params=None): """ Retrieve statistics on different operations happening on an index. @@ -586,11 +685,17 @@ class IndicesClient(NamespacedClient): :arg types: A comma-separated list of document types for the `indexing` index metric """ - return self.transport.perform_request('GET', _make_path(index, - '_stats', metric), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_stats", metric), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'operation_threading', 'verbose') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "operation_threading", + "verbose", + ) def segments(self, index=None, params=None): """ Provide low level segments information that a Lucene index (shard level) is built with. @@ -609,12 +714,25 @@ class IndicesClient(NamespacedClient): :arg operation_threading: TODO: ? :arg verbose: Includes detailed memory usage by Lucene., default False """ - return self.transport.perform_request('GET', _make_path(index, - '_segments'), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_segments"), params=params + ) - @query_params('all_shards', 'allow_no_indices', 'analyze_wildcard', - 'analyzer', 'default_operator', 'df', 'expand_wildcards', 'explain', - 'ignore_unavailable', 'lenient', 'operation_threading', 'q', 'rewrite') + @query_params( + "all_shards", + "allow_no_indices", + "analyze_wildcard", + "analyzer", + "default_operator", + "df", + "expand_wildcards", + "explain", + "ignore_unavailable", + "lenient", + "operation_threading", + "q", + "rewrite", + ) def validate_query(self, index=None, doc_type=None, body=None, params=None): """ Validate a potentially expensive query without executing it. @@ -651,12 +769,24 @@ class IndicesClient(NamespacedClient): :arg rewrite: Provide a more detailed explanation showing the actual Lucene query that will be executed. """ - return self.transport.perform_request('GET', _make_path(index, - doc_type, '_validate', 'query'), params=params, body=body) + return self.transport.perform_request( + "GET", + _make_path(index, doc_type, "_validate", "query"), + params=params, + body=body, + ) - @query_params('allow_no_indices', 'expand_wildcards', 'field_data', - 'fielddata', 'fields', 'ignore_unavailable', 'query', 'recycler', - 'request') + @query_params( + "allow_no_indices", + "expand_wildcards", + "field_data", + "fielddata", + "fields", + "ignore_unavailable", + "query", + "recycler", + "request", + ) def clear_cache(self, index=None, params=None): """ Clear either all caches or specific cached associated with one ore more indices. @@ -680,10 +810,11 @@ class IndicesClient(NamespacedClient): :arg request: Clear request cache :arg request_cache: Clear request cache """ - return self.transport.perform_request('POST', _make_path(index, - '_cache', 'clear'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_cache", "clear"), params=params + ) - @query_params('active_only', 'detailed') + @query_params("active_only", "detailed") def recovery(self, index=None, params=None): """ The indices recovery API provides insight into on-going shard @@ -698,11 +829,17 @@ class IndicesClient(NamespacedClient): :arg detailed: Whether to display detailed information about shard recovery, default False """ - return self.transport.perform_request('GET', _make_path(index, - '_recovery'), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_recovery"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'only_ancient_segments', 'wait_for_completion') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "only_ancient_segments", + "wait_for_completion", + ) def upgrade(self, index=None, params=None): """ Upgrade one or more indices to the latest format through an API. @@ -723,10 +860,11 @@ class IndicesClient(NamespacedClient): :arg wait_for_completion: Specify whether the request should block until the all segments are upgraded (default: false) """ - return self.transport.perform_request('POST', _make_path(index, - '_upgrade'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_upgrade"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable') + @query_params("allow_no_indices", "expand_wildcards", "ignore_unavailable") def get_upgrade(self, index=None, params=None): """ Monitor how much of one or more index is upgraded. @@ -743,10 +881,11 @@ class IndicesClient(NamespacedClient): :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed) """ - return self.transport.perform_request('GET', _make_path(index, - '_upgrade'), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_upgrade"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable') + @query_params("allow_no_indices", "expand_wildcards", "ignore_unavailable") def flush_synced(self, index=None, params=None): """ Perform a normal flush, then add a generated unique marker (sync_id) to all shards. @@ -763,11 +902,17 @@ class IndicesClient(NamespacedClient): :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed) """ - return self.transport.perform_request('POST', _make_path(index, - '_flush', 'synced'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_flush", "synced"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', - 'operation_threading', 'status') + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "operation_threading", + "status", + ) def shard_stores(self, index=None, params=None): """ Provides store information for shard copies of indices. Store @@ -791,12 +936,20 @@ class IndicesClient(NamespacedClient): to get store information for, valid choices are: 'green', 'yellow', 'red', 'all' """ - return self.transport.perform_request('GET', _make_path(index, - '_shard_stores'), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_shard_stores"), params=params + ) - @query_params('allow_no_indices', 'expand_wildcards', 'flush', - 'ignore_unavailable', 'max_num_segments', 'only_expunge_deletes', - 'operation_threading', 'wait_for_merge') + @query_params( + "allow_no_indices", + "expand_wildcards", + "flush", + "ignore_unavailable", + "max_num_segments", + "only_expunge_deletes", + "operation_threading", + "wait_for_merge", + ) def forcemerge(self, index=None, params=None): """ The force merge API allows to force merging of one or more indices @@ -827,10 +980,11 @@ class IndicesClient(NamespacedClient): expunge deleted documents :arg operation_threading: TODO: ? """ - return self.transport.perform_request('POST', _make_path(index, - '_forcemerge'), params=params) + return self.transport.perform_request( + "POST", _make_path(index, "_forcemerge"), params=params + ) - @query_params('master_timeout', 'timeout', 'wait_for_active_shards') + @query_params("master_timeout", "timeout", "wait_for_active_shards") def shrink(self, index, target, body=None, params=None): """ The shrink index API allows you to shrink an existing index into a new @@ -856,11 +1010,17 @@ class IndicesClient(NamespacedClient): for param in (index, target): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path(index, - '_shrink', target), params=params, body=body) + return self.transport.perform_request( + "PUT", _make_path(index, "_shrink", target), params=params, body=body + ) - @query_params('dry_run', 'master_timeout', 'timeout', - 'wait_for_active_shards', 'include_type_name') + @query_params( + "dry_run", + "master_timeout", + "timeout", + "wait_for_active_shards", + "include_type_name", + ) def rollover(self, alias, new_index=None, body=None, params=None): """ The rollover index API rolls an alias over to a new index when the @@ -887,5 +1047,6 @@ class IndicesClient(NamespacedClient): """ if alias in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'alias'.") - return self.transport.perform_request('POST', _make_path(alias, - '_rollover', new_index), params=params, body=body) + return self.transport.perform_request( + "POST", _make_path(alias, "_rollover", new_index), params=params, body=body + ) diff --git a/elasticsearch/client/nodes.py b/elasticsearch/client/nodes.py index 03f0f7ed..e009fa89 100644 --- a/elasticsearch/client/nodes.py +++ b/elasticsearch/client/nodes.py @@ -1,7 +1,19 @@ from .utils import NamespacedClient, query_params, _make_path + class NodesClient(NamespacedClient): - @query_params('flat_settings', 'timeout') + @query_params() + def reload_secure_settings(self, params=None): + """ + Reload any settings that have been marked as "reloadable" + ``_ + + """ + return self.transport.perform_request( + "POST", _make_path("_nodes", "reload_secure_settings"), params=params + ) + + @query_params("flat_settings", "timeout") def info(self, node_id=None, metric=None, params=None): """ The cluster nodes info API allows to retrieve one or more (or all) of @@ -17,11 +29,20 @@ class NodesClient(NamespacedClient): :arg flat_settings: Return settings in flat format (default: false) :arg timeout: Explicit operation timeout """ - return self.transport.perform_request('GET', _make_path('_nodes', - node_id, metric), params=params) + return self.transport.perform_request( + "GET", _make_path("_nodes", node_id, metric), params=params + ) - @query_params('completion_fields', 'fielddata_fields', 'fields', 'groups', - 'include_segment_file_sizes', 'level', 'timeout', 'types') + @query_params( + "completion_fields", + "fielddata_fields", + "fields", + "groups", + "include_segment_file_sizes", + "level", + "timeout", + "types", + ) def stats(self, node_id=None, metric=None, index_metric=None, params=None): """ The cluster nodes stats API allows to retrieve one or more (or all) of @@ -54,11 +75,15 @@ class NodesClient(NamespacedClient): :arg types: A comma-separated list of document types for the `indexing` index metric """ - return self.transport.perform_request('GET', _make_path('_nodes', - node_id, 'stats', metric, index_metric), params=params) + return self.transport.perform_request( + "GET", + _make_path("_nodes", node_id, "stats", metric, index_metric), + params=params, + ) - @query_params('type', 'ignore_idle_threads', 'interval', 'snapshots', - 'threads', 'timeout') + @query_params( + "type", "ignore_idle_threads", "interval", "snapshots", "threads", "timeout" + ) def hot_threads(self, node_id=None, params=None): """ An API allowing to get the current hot threads on each node in the cluster. @@ -80,12 +105,13 @@ class NodesClient(NamespacedClient): :arg timeout: Explicit operation timeout """ # avoid python reserved words - if params and 'type_' in params: - params['type'] = params.pop('type_') - return self.transport.perform_request('GET', _make_path('_cluster', - 'nodes', node_id, 'hotthreads'), params=params) + if params and "type_" in params: + params["type"] = params.pop("type_") + return self.transport.perform_request( + "GET", _make_path("_cluster", "nodes", node_id, "hotthreads"), params=params + ) - @query_params('human', 'timeout') + @query_params("human", "timeout") def usage(self, node_id=None, metric=None, params=None): """ The cluster nodes usage API allows to retrieve information on the usage @@ -101,5 +127,6 @@ class NodesClient(NamespacedClient): format., default False :arg timeout: Explicit operation timeout """ - return self.transport.perform_request('GET', _make_path('_nodes', - node_id, 'usage', metric), params=params) + return self.transport.perform_request( + "GET", _make_path("_nodes", node_id, "usage", metric), params=params + ) diff --git a/elasticsearch/client/utils.py b/elasticsearch/client/utils.py index eda4318f..a1e1ca9d 100644 --- a/elasticsearch/client/utils.py +++ b/elasticsearch/client/utils.py @@ -6,7 +6,8 @@ from functools import wraps from ..compat import string_types, quote_plus, PY2 # parts of URL to be omitted -SKIP_IN_PATH = (None, '', b'', [], ()) +SKIP_IN_PATH = (None, "", b"", [], ()) + def _escape(value): """ @@ -16,7 +17,7 @@ def _escape(value): # make sequences into comma-separated stings if isinstance(value, (list, tuple)): - value = ','.join(value) + value = ",".join(value) # dates and datetimes into isoformat elif isinstance(value, (date, datetime)): @@ -33,36 +34,43 @@ def _escape(value): # encode strings to utf-8 if isinstance(value, string_types): if PY2 and isinstance(value, unicode): - return value.encode('utf-8') + return value.encode("utf-8") if not PY2 and isinstance(value, str): - return value.encode('utf-8') + return value.encode("utf-8") return str(value) + def _make_path(*parts): """ Create a URL string from parts, omit all `None` values and empty strings. Convert lists nad tuples to comma separated values. """ - #TODO: maybe only allow some parts to be lists/tuples ? - return '/' + '/'.join( + # TODO: maybe only allow some parts to be lists/tuples ? + return "/" + "/".join( # preserve ',' and '*' in url for nicer URLs in logs - quote_plus(_escape(p), b',*') for p in parts if p not in SKIP_IN_PATH) + quote_plus(_escape(p), b",*") + for p in parts + if p not in SKIP_IN_PATH + ) + # parameters that apply to all methods -GLOBAL_PARAMS = ('pretty', 'human', 'error_trace', 'format', 'filter_path') +GLOBAL_PARAMS = ("pretty", "human", "error_trace", "format", "filter_path") + def query_params(*es_query_params): """ Decorator that pops all accepted parameters from method's kwargs and puts them in the params argument. """ + def _wrapper(func): @wraps(func) def _wrapped(*args, **kwargs): params = {} - if 'params' in kwargs: - params = kwargs.pop('params').copy() + if "params" in kwargs: + params = kwargs.pop("params").copy() for p in es_query_params + GLOBAL_PARAMS: if p in kwargs: v = kwargs.pop(p) @@ -70,11 +78,13 @@ def query_params(*es_query_params): params[p] = _escape(v) # don't treat ignore and request_timeout as other params to avoid escaping - for p in ('ignore', 'request_timeout'): + for p in ("ignore", "request_timeout"): if p in kwargs: params[p] = kwargs.pop(p) return func(*args, params=params, **kwargs) + return _wrapped + return _wrapper @@ -86,6 +96,7 @@ class NamespacedClient(object): def transport(self): return self.client.transport + class AddonClient(NamespacedClient): @classmethod def infect_client(cls, client): diff --git a/elasticsearch/client/xpack/__init__.py b/elasticsearch/client/xpack/__init__.py index 63b4155f..8c108c08 100644 --- a/elasticsearch/client/xpack/__init__.py +++ b/elasticsearch/client/xpack/__init__.py @@ -9,8 +9,9 @@ from .ml import MlClient from .migration import MigrationClient from .deprecation import DeprecationClient + class XPackClient(NamespacedClient): - namespace = 'xpack' + namespace = "xpack" def __init__(self, *args, **kwargs): super(XPackClient, self).__init__(*args, **kwargs) @@ -23,7 +24,7 @@ class XPackClient(NamespacedClient): self.migration = MigrationClient(self.client) self.deprecation = DeprecationClient(self.client) - @query_params('categories', 'human') + @query_params("categories", "human") def info(self, params=None): """ Retrieve information about xpack, including build number/timestamp and license status @@ -34,14 +35,13 @@ class XPackClient(NamespacedClient): :arg human: Presents additional info for humans (feature descriptions and X-Pack tagline) """ - return self.transport.perform_request('GET', '/_xpack', params=params) + return self.transport.perform_request("GET", "/_xpack", params=params) - @query_params('master_timeout') + @query_params("master_timeout") def usage(self, params=None): """ Retrieve information about xpack features usage :arg master_timeout: Specify timeout for watch write operation """ - return self.transport.perform_request('GET', '/_xpack/usage', - params=params) + return self.transport.perform_request("GET", "/_xpack/usage", params=params) diff --git a/elasticsearch/client/xpack/ccr.py b/elasticsearch/client/xpack/ccr.py new file mode 100644 index 00000000..c9153eaa --- /dev/null +++ b/elasticsearch/client/xpack/ccr.py @@ -0,0 +1,161 @@ +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + + +class CcrClient(NamespacedClient): + @query_params() + def delete_auto_follow_pattern(self, name, params=None): + """ + ``_ + + :arg name: The name of the auto follow pattern. + """ + if name in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'name'.") + return self.transport.perform_request( + "DELETE", _make_path("_ccr", "auto_follow", name), params=params + ) + + @query_params("wait_for_active_shards") + def follow(self, index, body, params=None): + """ + ``_ + + :arg index: The name of the follower index + :arg body: The name of the leader index and other optional ccr related + parameters + :arg wait_for_active_shards: Sets the number of shard copies that must + be active before returning. Defaults to 0. Set to `all` for all + shard copies, otherwise set to any non-negative value less than or + equal to the total number of copies for the shard (number of + replicas + 1), default '0' + """ + for param in (index, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", _make_path(index, "_ccr", "follow"), params=params, body=body + ) + + @query_params() + def follow_info(self, index=None, params=None): + """ + ``_ + + :arg index: A comma-separated list of index patterns; use `_all` to + perform the operation on all indices + """ + return self.transport.perform_request( + "GET", _make_path(index, "_ccr", "info"), params=params + ) + + @query_params() + def follow_stats(self, index, params=None): + """ + ``_ + + :arg index: A comma-separated list of index patterns; use `_all` to + perform the operation on all indices + """ + if index in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'index'.") + return self.transport.perform_request( + "GET", _make_path(index, "_ccr", "stats"), params=params + ) + + @query_params() + def forget_follower(self, index, body, params=None): + """ + ``_ + + :arg index: the name of the leader index for which specified follower + retention leases should be removed + :arg body: the name and UUID of the follower index, the name of the + cluster containing the follower index, and the alias from the + perspective of that cluster for the remote cluster containing the + leader index + """ + for param in (index, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "POST", + _make_path(index, "_ccr", "forget_follower"), + params=params, + body=body, + ) + + @query_params() + def get_auto_follow_pattern(self, name=None, params=None): + """ + ``_ + + :arg name: The name of the auto follow pattern. + """ + return self.transport.perform_request( + "GET", _make_path("_ccr", "auto_follow", name), params=params + ) + + @query_params() + def pause_follow(self, index, params=None): + """ + ``_ + + :arg index: The name of the follower index that should pause following + its leader index. + """ + if index in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'index'.") + return self.transport.perform_request( + "POST", _make_path(index, "_ccr", "pause_follow"), params=params + ) + + @query_params() + def put_auto_follow_pattern(self, name, body, params=None): + """ + ``_ + + :arg name: The name of the auto follow pattern. + :arg body: The specification of the auto follow pattern + """ + for param in (name, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", _make_path("_ccr", "auto_follow", name), params=params, body=body + ) + + @query_params() + def resume_follow(self, index, body=None, params=None): + """ + ``_ + + :arg index: The name of the follow index to resume following. + :arg body: The name of the leader index and other optional ccr related + parameters + """ + if index in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'index'.") + return self.transport.perform_request( + "POST", _make_path(index, "_ccr", "resume_follow"), params=params, body=body + ) + + @query_params() + def stats(self, params=None): + """ + ``_ + """ + return self.transport.perform_request("GET", "/_ccr/stats", params=params) + + @query_params() + def unfollow(self, index, params=None): + """ + ``_ + + :arg index: The name of the follower index that should be turned into a + regular index. + """ + if index in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'index'.") + return self.transport.perform_request( + "POST", _make_path(index, "_ccr", "unfollow"), params=params + ) diff --git a/elasticsearch/client/xpack/data_frame.py b/elasticsearch/client/xpack/data_frame.py new file mode 100644 index 00000000..8be6aae4 --- /dev/null +++ b/elasticsearch/client/xpack/data_frame.py @@ -0,0 +1,118 @@ +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + + +class Data_FrameClient(NamespacedClient): + @query_params() + def delete_data_frame_transform(self, transform_id, params=None): + """ + ``_ + + :arg transform_id: The id of the transform to delete + """ + if transform_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'transform_id'." + ) + return self.transport.perform_request( + "DELETE", + _make_path("_data_frame", "transforms", transform_id), + params=params, + ) + + @query_params("from_", "size") + def get_data_frame_transform(self, transform_id=None, params=None): + """ + ``_ + + :arg transform_id: The id or comma delimited list of id expressions of + the transforms to get, '_all' or '*' implies get all transforms + :arg from_: skips a number of transform configs, defaults to 0 + :arg size: specifies a max number of transforms to get, defaults to 100 + """ + return self.transport.perform_request( + "GET", _make_path("_data_frame", "transforms", transform_id), params=params + ) + + @query_params() + def get_data_frame_transform_stats(self, transform_id=None, params=None): + """ + ``_ + + :arg transform_id: The id of the transform for which to get stats. + '_all' or '*' implies all transforms + """ + return self.transport.perform_request( + "GET", + _make_path("_data_frame", "transforms", transform_id, "_stats"), + params=params, + ) + + @query_params() + def preview_data_frame_transform(self, body, params=None): + """ + ``_ + + :arg body: The definition for the data_frame transform to preview + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "POST", "/_data_frame/transforms/_preview", params=params, body=body + ) + + @query_params() + def put_data_frame_transform(self, transform_id, body, params=None): + """ + ``_ + + :arg transform_id: The id of the new transform. + :arg body: The data frame transform definition + """ + for param in (transform_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", + _make_path("_data_frame", "transforms", transform_id), + params=params, + body=body, + ) + + @query_params("timeout") + def start_data_frame_transform(self, transform_id, params=None): + """ + ``_ + + :arg transform_id: The id of the transform to start + :arg timeout: Controls the time to wait for the transform to start + """ + if transform_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'transform_id'." + ) + return self.transport.perform_request( + "POST", + _make_path("_data_frame", "transforms", transform_id, "_start"), + params=params, + ) + + @query_params("timeout", "wait_for_completion") + def stop_data_frame_transform(self, transform_id, params=None): + """ + ``_ + + :arg transform_id: The id of the transform to stop + :arg timeout: Controls the time to wait until the transform has stopped. + Default to 30 seconds + :arg wait_for_completion: Whether to wait for the transform to fully + stop before returning or not. Default to false + """ + if transform_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'transform_id'." + ) + return self.transport.perform_request( + "POST", + _make_path("_data_frame", "transforms", transform_id, "_stop"), + params=params, + ) diff --git a/elasticsearch/client/xpack/deprecation.py b/elasticsearch/client/xpack/deprecation.py index e9d7d76e..b8ca7510 100644 --- a/elasticsearch/client/xpack/deprecation.py +++ b/elasticsearch/client/xpack/deprecation.py @@ -1,5 +1,6 @@ from ..utils import NamespacedClient, query_params, _make_path + class DeprecationClient(NamespacedClient): @query_params() def info(self, index=None, params=None): @@ -8,6 +9,8 @@ class DeprecationClient(NamespacedClient): :arg index: Index pattern """ - return self.transport.perform_request('GET', _make_path(index, '_xpack', - 'migration', 'deprecations'), params=params) - + return self.transport.perform_request( + "GET", + _make_path(index, "_xpack", "migration", "deprecations"), + params=params, + ) diff --git a/elasticsearch/client/xpack/graph.py b/elasticsearch/client/xpack/graph.py index 62322867..f847151d 100644 --- a/elasticsearch/client/xpack/graph.py +++ b/elasticsearch/client/xpack/graph.py @@ -1,7 +1,8 @@ -from ..utils import NamespacedClient, query_params, _make_path +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class GraphClient(NamespacedClient): - @query_params('routing', 'timeout') + @query_params("routing", "timeout") def explore(self, index=None, doc_type=None, body=None, params=None): """ ``_ @@ -14,5 +15,9 @@ class GraphClient(NamespacedClient): :arg routing: Specific routing value :arg timeout: Explicit operation timeout """ - return self.transport.perform_request('GET', _make_path(index, doc_type, - '_xpack', 'graph', '_explore'), params=params, body=body) + return self.transport.perform_request( + "GET", + _make_path(index, doc_type, "_graph", "explore"), + params=params, + body=body, + ) diff --git a/elasticsearch/client/xpack/ilm.py b/elasticsearch/client/xpack/ilm.py new file mode 100644 index 00000000..ea0c91da --- /dev/null +++ b/elasticsearch/client/xpack/ilm.py @@ -0,0 +1,104 @@ +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + + +class IlmClient(NamespacedClient): + @query_params() + def delete_lifecycle(self, policy=None, params=None): + """ + ``_ + + :arg policy: The name of the index lifecycle policy + """ + return self.transport.perform_request( + "DELETE", _make_path("_ilm", "policy", policy), params=params + ) + + @query_params() + def explain_lifecycle(self, index=None, params=None): + """ + ``_ + + :arg index: The name of the index to explain + """ + return self.transport.perform_request( + "GET", _make_path(index, "_ilm", "explain"), params=params + ) + + @query_params() + def get_lifecycle(self, policy=None, params=None): + """ + ``_ + + :arg policy: The name of the index lifecycle policy + """ + return self.transport.perform_request( + "GET", _make_path("_ilm", "policy", policy), params=params + ) + + @query_params() + def get_status(self, params=None): + """ + ``_ + """ + return self.transport.perform_request("GET", "/_ilm/status", params=params) + + @query_params() + def move_to_step(self, index=None, body=None, params=None): + """ + ``_ + + :arg index: The name of the index whose lifecycle step is to change + :arg body: The new lifecycle step to move to + """ + return self.transport.perform_request( + "POST", _make_path("_ilm", "move", index), params=params, body=body + ) + + @query_params() + def put_lifecycle(self, policy=None, body=None, params=None): + """ + ``_ + + :arg policy: The name of the index lifecycle policy + :arg body: The lifecycle policy definition to register + """ + return self.transport.perform_request( + "PUT", _make_path("_ilm", "policy", policy), params=params, body=body + ) + + @query_params() + def remove_policy(self, index=None, params=None): + """ + ``_ + + :arg index: The name of the index to remove policy on + """ + return self.transport.perform_request( + "POST", _make_path(index, "_ilm", "remove"), params=params + ) + + @query_params() + def retry(self, index=None, params=None): + """ + ``_ + + :arg index: The name of the indices (comma-separated) whose failed + lifecycle step is to be retry + """ + return self.transport.perform_request( + "POST", _make_path(index, "_ilm", "retry"), params=params + ) + + @query_params() + def start(self, params=None): + """ + ``_ + """ + return self.transport.perform_request("POST", "/_ilm/start", params=params) + + @query_params() + def stop(self, params=None): + """ + ``_ + """ + return self.transport.perform_request("POST", "/_ilm/stop", params=params) diff --git a/elasticsearch/client/xpack/indices.py b/elasticsearch/client/xpack/indices.py new file mode 100644 index 00000000..c4e48807 --- /dev/null +++ b/elasticsearch/client/xpack/indices.py @@ -0,0 +1,67 @@ +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + + +class IndicesClient(NamespacedClient): + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "master_timeout", + "timeout", + "wait_for_active_shards", + ) + def freeze(self, index, params=None): + """ + ``_ + + :arg index: The name of the index to freeze + :arg allow_no_indices: Whether to ignore if a wildcard indices + expression resolves into no concrete indices. (This includes `_all` + string or when no indices have been specified) + :arg expand_wildcards: Whether to expand wildcard expression to concrete + indices that are open, closed or both., default 'closed', valid + choices are: 'open', 'closed', 'none', 'all' + :arg ignore_unavailable: Whether specified concrete indices should be + ignored when unavailable (missing or closed) + :arg master_timeout: Specify timeout for connection to master + :arg timeout: Explicit operation timeout + :arg wait_for_active_shards: Sets the number of active shards to wait + for before the operation returns. + """ + if index in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'index'.") + return self.transport.perform_request( + "POST", _make_path(index, "_freeze"), params=params + ) + + @query_params( + "allow_no_indices", + "expand_wildcards", + "ignore_unavailable", + "master_timeout", + "timeout", + "wait_for_active_shards", + ) + def unfreeze(self, index, params=None): + """ + ``_ + + :arg index: The name of the index to unfreeze + :arg allow_no_indices: Whether to ignore if a wildcard indices + expression resolves into no concrete indices. (This includes `_all` + string or when no indices have been specified) + :arg expand_wildcards: Whether to expand wildcard expression to concrete + indices that are open, closed or both., default 'closed', valid + choices are: 'open', 'closed', 'none', 'all' + :arg ignore_unavailable: Whether specified concrete indices should be + ignored when unavailable (missing or closed) + :arg master_timeout: Specify timeout for connection to master + :arg timeout: Explicit operation timeout + :arg wait_for_active_shards: Sets the number of active shards to wait + for before the operation returns. + """ + if index in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'index'.") + return self.transport.perform_request( + "POST", _make_path(index, "_unfreeze"), params=params + ) diff --git a/elasticsearch/client/xpack/license.py b/elasticsearch/client/xpack/license.py index 371891ac..071c2382 100644 --- a/elasticsearch/client/xpack/license.py +++ b/elasticsearch/client/xpack/license.py @@ -1,37 +1,76 @@ from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class LicenseClient(NamespacedClient): @query_params() def delete(self, params=None): """ - ``_ """ - return self.transport.perform_request('DELETE', '/_xpack/license', - params=params) + return self.transport.perform_request("DELETE", "/_license", params=params) - @query_params('local') + @query_params("local") def get(self, params=None): """ - ``_ :arg local: Return local information, do not retrieve the state from master node (default: false) """ - return self.transport.perform_request('GET', '/_xpack/license', - params=params) + return self.transport.perform_request("GET", "/_license", params=params) - @query_params('acknowledge') + @query_params() + def get_basic_status(self, params=None): + """ + ``_ + """ + return self.transport.perform_request( + "GET", "/_license/basic_status", params=params + ) + + @query_params() + def get_trial_status(self, params=None): + """ + ``_ + """ + return self.transport.perform_request( + "GET", "/_license/trial_status", params=params + ) + + @query_params("acknowledge") def post(self, body=None, params=None): """ - ``_ :arg body: licenses to be installed :arg acknowledge: whether the user has acknowledged acknowledge messages (default: false) """ - return self.transport.perform_request('PUT', '/_xpack/license', - params=params, body=body) + return self.transport.perform_request( + "PUT", "/_license", params=params, body=body + ) + @query_params("acknowledge") + def post_start_basic(self, params=None): + """ + ``_ + + :arg acknowledge: whether the user has acknowledged acknowledge messages + (default: false) + """ + return self.transport.perform_request( + "POST", "/_license/start_basic", params=params + ) + + @query_params("acknowledge", "doc_type") + def post_start_trial(self, params=None): + """ + ``_ + + :arg acknowledge: whether the user has acknowledged acknowledge messages + (default: false) + :arg doc_type: The type of trial license to generate (default: "trial") + """ + return self.transport.perform_request( + "POST", "/_license/start_trial", params=params + ) diff --git a/elasticsearch/client/xpack/migration.py b/elasticsearch/client/xpack/migration.py index 50c17bd3..589eb569 100644 --- a/elasticsearch/client/xpack/migration.py +++ b/elasticsearch/client/xpack/migration.py @@ -1,36 +1,14 @@ from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class MigrationClient(NamespacedClient): - @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable') - def get_assistance(self, index=None, params=None): + @query_params() + def deprecations(self, index=None, params=None): """ - ``_ + ``_ - :arg index: A comma-separated list of index names; use `_all` or empty - string to perform the operation on all indices - :arg allow_no_indices: Whether to ignore if a wildcard indices - expression resolves into no concrete indices. (This includes `_all` - string or when no indices have been specified) - :arg expand_wildcards: Whether to expand wildcard expression to concrete - indices that are open, closed or both., default 'open', valid - choices are: 'open', 'closed', 'none', 'all' - :arg ignore_unavailable: Whether specified concrete indices should be - ignored when unavailable (missing or closed) + :arg index: Index pattern """ - return self.transport.perform_request('GET', _make_path('_xpack', - 'migration', 'assistance', index), params=params) - - @query_params('wait_for_completion') - def upgrade(self, index, params=None): - """ - - ``_ - - :arg index: The name of the index - :arg wait_for_completion: Should the request block until the upgrade - operation is completed, default True - """ - if index in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'index'.") - return self.transport.perform_request('POST', _make_path('_xpack', - 'migration', 'upgrade', index), params=params) + return self.transport.perform_request( + "GET", _make_path(index, "_migration", "deprecations"), params=params + ) diff --git a/elasticsearch/client/xpack/ml.py b/elasticsearch/client/xpack/ml.py index 70b060a9..9fcb7347 100644 --- a/elasticsearch/client/xpack/ml.py +++ b/elasticsearch/client/xpack/ml.py @@ -1,249 +1,235 @@ from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class MlClient(NamespacedClient): - @query_params('from_', 'size') - def get_filters(self, filter_id=None, params=None): + @query_params("allow_no_jobs", "force", "timeout") + def close_job(self, job_id, body=None, params=None): """ + ``_ - :arg filter_id: The ID of the filter to fetch - :arg from_: skips a number of filters - :arg size: specifies a max number of filters to get - """ - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'filters', filter_id), params=params) - - @query_params() - def get_datafeeds(self, datafeed_id=None, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeeds to fetch - """ - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'datafeeds', datafeed_id), params=params) - - @query_params() - def get_datafeed_stats(self, datafeed_id=None, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeeds stats to fetch - """ - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'datafeeds', datafeed_id, '_stats'), params=params) - - @query_params('anomaly_score', 'desc', 'end', 'exclude_interim', 'expand', - 'from_', 'size', 'sort', 'start') - def get_buckets(self, job_id, timestamp=None, body=None, params=None): - """ - - ``_ - - :arg job_id: ID of the job to get bucket results from - :arg timestamp: The timestamp of the desired single bucket result - :arg body: Bucket selection details if not provided in URI - :arg anomaly_score: Filter for the most anomalous buckets - :arg desc: Set the sort direction - :arg end: End time filter for buckets - :arg exclude_interim: Exclude interim results - :arg expand: Include anomaly records - :arg from_: skips a number of buckets - :arg size: specifies a max number of buckets to get - :arg sort: Sort buckets by a particular field - :arg start: Start time filter for buckets + :arg job_id: The name of the job to close + :arg body: The URL params optionally sent in the body + :arg allow_no_jobs: Whether to ignore if a wildcard expression matches + no jobs. (This includes `_all` string or when no jobs have been + specified) + :arg force: True if the job should be forcefully closed + :arg timeout: Controls the time to wait until a job has closed. Default + to 30 minutes """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, 'results', 'buckets', timestamp), - params=params, body=body) - - @query_params('reset_end', 'reset_start') - def post_data(self, job_id, body, params=None): - """ - - ``_ - - :arg job_id: The name of the job receiving the data - :arg body: The data to process - :arg reset_end: Optional parameter to specify the end of the bucket - resetting range - :arg reset_start: Optional parameter to specify the start of the bucket - resetting range - """ - for param in (job_id, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, '_data'), params=params, - body=self._bulk_body(body)) - - @query_params('force', 'timeout') - def stop_datafeed(self, datafeed_id, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeed to stop - :arg force: True if the datafeed should be forcefully stopped. - :arg timeout: Controls the time to wait until a datafeed has stopped. - Default to 20 seconds - """ - if datafeed_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'datafeed_id'.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'datafeeds', datafeed_id, '_stop'), params=params) + return self.transport.perform_request( + "POST", + _make_path("_ml", "anomaly_detectors", job_id, "_close"), + params=params, + body=body, + ) @query_params() - def get_jobs(self, job_id=None, params=None): + def delete_calendar(self, calendar_id, params=None): """ + `<>`_ - ``_ - - :arg job_id: The ID of the jobs to fetch + :arg calendar_id: The ID of the calendar to delete """ - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id), params=params) + if calendar_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'calendar_id'." + ) + return self.transport.perform_request( + "DELETE", _make_path("_ml", "calendars", calendar_id), params=params + ) + + @query_params() + def delete_calendar_event(self, calendar_id, event_id, params=None): + """ + `<>`_ + + :arg calendar_id: The ID of the calendar to modify + :arg event_id: The ID of the event to remove from the calendar + """ + for param in (calendar_id, event_id): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "DELETE", + _make_path("_ml", "calendars", calendar_id, "events", event_id), + params=params, + ) + + @query_params() + def delete_calendar_job(self, calendar_id, job_id, params=None): + """ + `<>`_ + + :arg calendar_id: The ID of the calendar to modify + :arg job_id: The ID of the job to remove from the calendar + """ + for param in (calendar_id, job_id): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "DELETE", + _make_path("_ml", "calendars", calendar_id, "jobs", job_id), + params=params, + ) + + @query_params("force") + def delete_datafeed(self, datafeed_id, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeed to delete + :arg force: True if the datafeed should be forcefully deleted + """ + if datafeed_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'datafeed_id'." + ) + return self.transport.perform_request( + "DELETE", _make_path("_ml", "datafeeds", datafeed_id), params=params + ) @query_params() def delete_expired_data(self, params=None): """ + `<>`_ """ - return self.transport.perform_request('DELETE', - '/_xpack/ml/_delete_expired_data', params=params) + return self.transport.perform_request( + "DELETE", "/_ml/_delete_expired_data", params=params + ) @query_params() - def put_job(self, job_id, body, params=None): + def delete_filter(self, filter_id, params=None): """ + `<>`_ - ``_ - - :arg job_id: The ID of the job to create - :arg body: The job + :arg filter_id: The ID of the filter to delete """ - for param in (job_id, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id), params=params, body=body) + if filter_id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'filter_id'.") + return self.transport.perform_request( + "DELETE", _make_path("_ml", "filters", filter_id), params=params + ) - @query_params() - def validate_detector(self, body, params=None): + @query_params("allow_no_forecasts", "timeout") + def delete_forecast(self, job_id, forecast_id=None, params=None): """ + ``_ - :arg body: The detector - """ - if body in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('POST', - '/_xpack/ml/anomaly_detectors/_validate/detector', params=params, - body=body) - - @query_params('end', 'start', 'timeout') - def start_datafeed(self, datafeed_id, body=None, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeed to start - :arg body: The start datafeed parameters - :arg end: The end time when the datafeed should stop. When not set, the - datafeed continues in real time - :arg start: The start time from where the datafeed should begin - :arg timeout: Controls the time to wait until a datafeed has started. - Default to 20 seconds - """ - if datafeed_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'datafeed_id'.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'datafeeds', datafeed_id, '_start'), params=params, body=body) - - @query_params('desc', 'end', 'exclude_interim', 'from_', 'record_score', - 'size', 'sort', 'start') - def get_records(self, job_id, body=None, params=None): - """ - - ``_ - - :arg job_id: None - :arg body: Record selection criteria - :arg desc: Set the sort direction - :arg end: End time filter for records - :arg exclude_interim: Exclude interim results - :arg from_: skips a number of records - :arg record_score: - :arg size: specifies a max number of records to get - :arg sort: Sort records by a particular field - :arg start: Start time filter for records + :arg job_id: The ID of the job from which to delete forecasts + :arg forecast_id: The ID of the forecast to delete, can be comma + delimited list. Leaving blank implies `_all` + :arg allow_no_forecasts: Whether to ignore if `_all` matches no + forecasts + :arg timeout: Controls the time to wait until the forecast(s) are + deleted. Default to 30 seconds """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, 'results', 'records'), params=params, - body=body) + return self.transport.perform_request( + "DELETE", + _make_path("_ml", "anomaly_detectors", job_id, "_forecast", forecast_id), + params=params, + ) + + @query_params("force", "wait_for_completion") + def delete_job(self, job_id, params=None): + """ + ``_ + + :arg job_id: The ID of the job to delete + :arg force: True if the job should be forcefully deleted, default False + :arg wait_for_completion: Should this request wait until the operation + has completed before returning, default True + """ + if job_id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'job_id'.") + return self.transport.perform_request( + "DELETE", _make_path("_ml", "anomaly_detectors", job_id), params=params + ) @query_params() - def update_job(self, job_id, body, params=None): + def delete_model_snapshot(self, job_id, snapshot_id, params=None): """ + ``_ - ``_ - - :arg job_id: The ID of the job to create - :arg body: The job update settings + :arg job_id: The ID of the job to fetch + :arg snapshot_id: The ID of the snapshot to delete """ - for param in (job_id, body): + for param in (job_id, snapshot_id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, '_update'), params=params, body=body) + return self.transport.perform_request( + "DELETE", + _make_path( + "_ml", "anomaly_detectors", job_id, "model_snapshots", snapshot_id + ), + params=params, + ) - @query_params() - def put_filter(self, filter_id, body, params=None): + @query_params( + "charset", + "column_names", + "delimiter", + "explain", + "format", + "grok_pattern", + "has_header_row", + "lines_to_sample", + "quote", + "should_trim_fields", + "timeout", + "timestamp_field", + "timestamp_format", + ) + def find_file_structure(self, body, params=None): """ + ``_ - :arg filter_id: The ID of the filter to create - :arg body: The filter details + :arg body: The contents of the file to be analyzed + :arg charset: Optional parameter to specify the character set of the + file + :arg column_names: Optional parameter containing a comma separated list + of the column names for a delimited file + :arg delimiter: Optional parameter to specify the delimiter character + for a delimited file - must be a single character + :arg explain: Whether to include a commentary on how the structure was + derived, default False + :arg format: Optional parameter to specify the high level file format, + valid choices are: 'ndjson', 'xml', 'delimited', + 'semi_structured_text' + :arg grok_pattern: Optional parameter to specify the Grok pattern that + should be used to extract fields from messages in a semi-structured + text file + :arg has_header_row: Optional parameter to specify whether a delimited + file includes the column names in its first row + :arg lines_to_sample: How many lines of the file should be included in + the analysis, default 1000 + :arg quote: Optional parameter to specify the quote character for a + delimited file - must be a single character + :arg should_trim_fields: Optional parameter to specify whether the + values between delimiters in a delimited file should have whitespace + trimmed from them + :arg timeout: Timeout after which the analysis will be aborted, default + '25s' + :arg timestamp_field: Optional parameter to specify the timestamp field + in the file + :arg timestamp_format: Optional parameter to specify the timestamp + format in the file - may be either a Joda or Java time format """ - for param in (filter_id, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_xpack', 'ml', - 'filters', filter_id), params=params, body=body) + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "POST", + "/_ml/find_file_structure", + params=params, + body=self._bulk_body(body), + ) - @query_params() - def update_datafeed(self, datafeed_id, body, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeed to update - :arg body: The datafeed update settings - """ - for param in (datafeed_id, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'datafeeds', datafeed_id, '_update'), params=params, body=body) - - @query_params() - def preview_datafeed(self, datafeed_id, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeed to preview - """ - if datafeed_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'datafeed_id'.") - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'datafeeds', datafeed_id, '_preview'), params=params) - - @query_params('advance_time', 'calc_interim', 'end', 'skip_time', 'start') + @query_params("advance_time", "calc_interim", "end", "skip_time", "start") def flush_job(self, job_id, body=None, params=None): """ - ``_ :arg job_id: The name of the job to flush @@ -261,110 +247,108 @@ class MlClient(NamespacedClient): """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, '_flush'), params=params, body=body) + return self.transport.perform_request( + "POST", + _make_path("_ml", "anomaly_detectors", job_id, "_flush"), + params=params, + body=body, + ) - @query_params('force', 'timeout') - def close_job(self, job_id, params=None): + @query_params("duration", "expires_in") + def forecast(self, job_id, params=None): """ + `<>`_ - ``_ - - :arg job_id: The name of the job to close - :arg force: True if the job should be forcefully closed - :arg timeout: Controls the time to wait until a job has closed. Default - to 30 minutes + :arg job_id: The ID of the job to forecast for + :arg duration: The duration of the forecast + :arg expires_in: The time interval after which the forecast expires. + Expired forecasts will be deleted at the first opportunity. """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, '_close'), params=params) + return self.transport.perform_request( + "POST", + _make_path("_ml", "anomaly_detectors", job_id, "_forecast"), + params=params, + ) - @query_params() - def open_job(self, job_id, params=None): + @query_params( + "anomaly_score", + "desc", + "end", + "exclude_interim", + "expand", + "from_", + "size", + "sort", + "start", + ) + def get_buckets(self, job_id, timestamp=None, body=None, params=None): """ + ``_ - ``_ - - :arg job_id: The ID of the job to open + :arg job_id: ID of the job to get bucket results from + :arg timestamp: The timestamp of the desired single bucket result + :arg body: Bucket selection details if not provided in URI + :arg anomaly_score: Filter for the most anomalous buckets + :arg desc: Set the sort direction + :arg end: End time filter for buckets + :arg exclude_interim: Exclude interim results + :arg expand: Include anomaly records + :arg from_: skips a number of buckets + :arg size: specifies a max number of buckets to get + :arg sort: Sort buckets by a particular field + :arg start: Start time filter for buckets """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, '_open'), params=params) + return self.transport.perform_request( + "GET", + _make_path( + "_ml", "anomaly_detectors", job_id, "results", "buckets", timestamp + ), + params=params, + body=body, + ) - @query_params('force') - def delete_job(self, job_id, params=None): + @query_params("end", "from_", "job_id", "size", "start") + def get_calendar_events(self, calendar_id, params=None): """ + `<>`_ - ``_ - - :arg job_id: The ID of the job to delete - :arg force: True if the job should be forcefully deleted + :arg calendar_id: The ID of the calendar containing the events + :arg end: Get events before this time + :arg from_: Skips a number of events + :arg job_id: Get events for the job. When this option is used + calendar_id must be '_all' + :arg size: Specifies a max number of events to get + :arg start: Get events after this time """ - if job_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'ml', 'anomaly_detectors', job_id), params=params) + if calendar_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'calendar_id'." + ) + return self.transport.perform_request( + "GET", _make_path("_ml", "calendars", calendar_id, "events"), params=params + ) - @query_params('duration', 'expires_in') - def forecast_job(self, job_id, params=None): + @query_params("from_", "size") + def get_calendars(self, calendar_id=None, body=None, params=None): """ + `<>`_ - ``_ - - :arg job_id: The name of the job to close - :arg duration: A period of time that indicates how far into the future to forecast - :arg expires_in: The period of time that forecast results are retained. + :arg calendar_id: The ID of the calendar to fetch + :arg body: The from and size parameters optionally sent in the body + :arg from_: skips a number of calendars + :arg size: specifies a max number of calendars to get """ - if job_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, '_forecast'), params=params) + return self.transport.perform_request( + "GET", _make_path("_ml", "calendars", calendar_id), params=params, body=body + ) - @query_params() - def update_model_snapshot(self, job_id, snapshot_id, body, params=None): - """ - - ``_ - - :arg job_id: The ID of the job to fetch - :arg snapshot_id: The ID of the snapshot to update - :arg body: The model snapshot properties to update - """ - for param in (job_id, snapshot_id, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, 'model_snapshots', snapshot_id, - '_update'), params=params, body=body) - - @query_params() - def delete_filter(self, filter_id, params=None): - """ - - :arg filter_id: The ID of the filter to delete - """ - if filter_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'filter_id'.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'ml', 'filters', filter_id), params=params) - - @query_params() - def validate(self, body, params=None): - """ - - :arg body: The job config - """ - if body in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('POST', - '/_xpack/ml/anomaly_detectors/_validate', params=params, body=body) - - @query_params('from_', 'size') + @query_params("from_", "size") def get_categories(self, job_id, category_id=None, body=None, params=None): """ - ``_ :arg job_id: The name of the job @@ -375,15 +359,68 @@ class MlClient(NamespacedClient): """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, 'results', 'categories', category_id), - params=params, body=body) + return self.transport.perform_request( + "GET", + _make_path( + "_ml", "anomaly_detectors", job_id, "results", "categories", category_id + ), + params=params, + body=body, + ) - @query_params('desc', 'end', 'exclude_interim', 'from_', 'influencer_score', - 'size', 'sort', 'start') + @query_params("allow_no_datafeeds") + def get_datafeed_stats(self, datafeed_id=None, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeeds stats to fetch + :arg allow_no_datafeeds: Whether to ignore if a wildcard expression + matches no datafeeds. (This includes `_all` string or when no + datafeeds have been specified) + """ + return self.transport.perform_request( + "GET", _make_path("_ml", "datafeeds", datafeed_id, "_stats"), params=params + ) + + @query_params("allow_no_datafeeds") + def get_datafeeds(self, datafeed_id=None, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeeds to fetch + :arg allow_no_datafeeds: Whether to ignore if a wildcard expression + matches no datafeeds. (This includes `_all` string or when no + datafeeds have been specified) + """ + return self.transport.perform_request( + "GET", _make_path("_ml", "datafeeds", datafeed_id), params=params + ) + + @query_params("from_", "size") + def get_filters(self, filter_id=None, params=None): + """ + `<>`_ + + :arg filter_id: The ID of the filter to fetch + :arg from_: skips a number of filters + :arg size: specifies a max number of filters to get + """ + return self.transport.perform_request( + "GET", _make_path("_ml", "filters", filter_id), params=params + ) + + @query_params( + "desc", + "end", + "exclude_interim", + "from_", + "influencer_score", + "size", + "sort", + "start", + ) def get_influencers(self, job_id, body=None, params=None): """ - ``_ :arg job_id: None @@ -400,73 +437,46 @@ class MlClient(NamespacedClient): """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, 'results', 'influencers'), - params=params, body=body) + return self.transport.perform_request( + "GET", + _make_path("_ml", "anomaly_detectors", job_id, "results", "influencers"), + params=params, + body=body, + ) - @query_params() - def put_datafeed(self, datafeed_id, body, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeed to create - :arg body: The datafeed config - """ - for param in (datafeed_id, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_xpack', 'ml', - 'datafeeds', datafeed_id), params=params, body=body) - - @query_params('force') - def delete_datafeed(self, datafeed_id, params=None): - """ - - ``_ - - :arg datafeed_id: The ID of the datafeed to delete - :arg force: True if the datafeed should be forcefully deleted - """ - if datafeed_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'datafeed_id'.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'ml', 'datafeeds', datafeed_id), params=params) - - @query_params() + @query_params("allow_no_jobs") def get_job_stats(self, job_id=None, params=None): """ - ``_ :arg job_id: The ID of the jobs stats to fetch + :arg allow_no_jobs: Whether to ignore if a wildcard expression matches + no jobs. (This includes `_all` string or when no jobs have been + specified) """ - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, '_stats'), params=params) + return self.transport.perform_request( + "GET", + _make_path("_ml", "anomaly_detectors", job_id, "_stats"), + params=params, + ) - @query_params('delete_intervening_results') - def revert_model_snapshot(self, job_id, snapshot_id, body=None, params=None): + @query_params("allow_no_jobs") + def get_jobs(self, job_id=None, params=None): """ + ``_ - ``_ - - :arg job_id: The ID of the job to fetch - :arg snapshot_id: The ID of the snapshot to revert to - :arg body: Reversion options - :arg delete_intervening_results: Should we reset the results back to the - time of the snapshot? + :arg job_id: The ID of the jobs to fetch + :arg allow_no_jobs: Whether to ignore if a wildcard expression matches + no jobs. (This includes `_all` string or when no jobs have been + specified) """ - for param in (job_id, snapshot_id): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('POST', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, 'model_snapshots', snapshot_id, - '_revert'), params=params, body=body) + return self.transport.perform_request( + "GET", _make_path("_ml", "anomaly_detectors", job_id), params=params + ) - @query_params('desc', 'end', 'from_', 'size', 'sort', 'start') + @query_params("desc", "end", "from_", "size", "sort", "start") def get_model_snapshots(self, job_id, snapshot_id=None, body=None, params=None): """ - ``_ :arg job_id: The ID of the job to fetch @@ -482,23 +492,441 @@ class MlClient(NamespacedClient): """ if job_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'job_id'.") - return self.transport.perform_request('GET', _make_path('_xpack', 'ml', - 'anomaly_detectors', job_id, 'model_snapshots', snapshot_id), - params=params, body=body) + return self.transport.perform_request( + "GET", + _make_path( + "_ml", "anomaly_detectors", job_id, "model_snapshots", snapshot_id + ), + params=params, + body=body, + ) + + @query_params( + "allow_no_jobs", + "bucket_span", + "end", + "exclude_interim", + "overall_score", + "start", + "top_n", + ) + def get_overall_buckets(self, job_id, body=None, params=None): + """ + ``_ + + :arg job_id: The job IDs for which to calculate overall bucket results + :arg body: Overall bucket selection details if not provided in URI + :arg allow_no_jobs: Whether to ignore if a wildcard expression matches + no jobs. (This includes `_all` string or when no jobs have been + specified) + :arg bucket_span: The span of the overall buckets. Defaults to the + longest job bucket_span + :arg end: Returns overall buckets with timestamps earlier than this time + :arg exclude_interim: If true overall buckets that include interim + buckets will be excluded + :arg overall_score: Returns overall buckets with overall scores higher + than this value + :arg start: Returns overall buckets with timestamps after this time + :arg top_n: The number of top job bucket scores to be used in the + overall_score calculation + """ + if job_id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'job_id'.") + return self.transport.perform_request( + "GET", + _make_path( + "_ml", "anomaly_detectors", job_id, "results", "overall_buckets" + ), + params=params, + body=body, + ) + + @query_params( + "desc", + "end", + "exclude_interim", + "from_", + "record_score", + "size", + "sort", + "start", + ) + def get_records(self, job_id, body=None, params=None): + """ + ``_ + + :arg job_id: None + :arg body: Record selection criteria + :arg desc: Set the sort direction + :arg end: End time filter for records + :arg exclude_interim: Exclude interim results + :arg from_: skips a number of records + :arg record_score: + :arg size: specifies a max number of records to get + :arg sort: Sort records by a particular field + :arg start: Start time filter for records + """ + if job_id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'job_id'.") + return self.transport.perform_request( + "GET", + _make_path("_ml", "anomaly_detectors", job_id, "results", "records"), + params=params, + body=body, + ) @query_params() - def delete_model_snapshot(self, job_id, snapshot_id, params=None): + def info(self, params=None): """ + `<>`_ + """ + return self.transport.perform_request("GET", "/_ml/info", params=params) - ``_ + @query_params() + def open_job(self, job_id, params=None): + """ + ``_ + + :arg job_id: The ID of the job to open + """ + if job_id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'job_id'.") + return self.transport.perform_request( + "POST", + _make_path("_ml", "anomaly_detectors", job_id, "_open"), + params=params, + ) + + @query_params() + def post_calendar_events(self, calendar_id, body, params=None): + """ + `<>`_ + + :arg calendar_id: The ID of the calendar to modify + :arg body: A list of events + """ + for param in (calendar_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "POST", + _make_path("_ml", "calendars", calendar_id, "events"), + params=params, + body=body, + ) + + @query_params("reset_end", "reset_start") + def post_data(self, job_id, body, params=None): + """ + ``_ + + :arg job_id: The name of the job receiving the data + :arg body: The data to process + :arg reset_end: Optional parameter to specify the end of the bucket + resetting range + :arg reset_start: Optional parameter to specify the start of the bucket + resetting range + """ + for param in (job_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "POST", + _make_path("_ml", "anomaly_detectors", job_id, "_data"), + params=params, + body=self._bulk_body(body), + ) + + @query_params() + def preview_datafeed(self, datafeed_id, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeed to preview + """ + if datafeed_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'datafeed_id'." + ) + return self.transport.perform_request( + "GET", + _make_path("_ml", "datafeeds", datafeed_id, "_preview"), + params=params, + ) + + @query_params() + def put_calendar(self, calendar_id, body=None, params=None): + """ + `<>`_ + + :arg calendar_id: The ID of the calendar to create + :arg body: The calendar details + """ + if calendar_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'calendar_id'." + ) + return self.transport.perform_request( + "PUT", _make_path("_ml", "calendars", calendar_id), params=params, body=body + ) + + @query_params() + def put_calendar_job(self, calendar_id, job_id, params=None): + """ + `<>`_ + + :arg calendar_id: The ID of the calendar to modify + :arg job_id: The ID of the job to add to the calendar + """ + for param in (calendar_id, job_id): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", + _make_path("_ml", "calendars", calendar_id, "jobs", job_id), + params=params, + ) + + @query_params() + def put_datafeed(self, datafeed_id, body, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeed to create + :arg body: The datafeed config + """ + for param in (datafeed_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", _make_path("_ml", "datafeeds", datafeed_id), params=params, body=body + ) + + @query_params() + def put_filter(self, filter_id, body, params=None): + """ + `<>`_ + + :arg filter_id: The ID of the filter to create + :arg body: The filter details + """ + for param in (filter_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", _make_path("_ml", "filters", filter_id), params=params, body=body + ) + + @query_params() + def put_job(self, job_id, body, params=None): + """ + ``_ + + :arg job_id: The ID of the job to create + :arg body: The job + """ + for param in (job_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", + _make_path("_ml", "anomaly_detectors", job_id), + params=params, + body=body, + ) + + @query_params("delete_intervening_results") + def revert_model_snapshot(self, job_id, snapshot_id, body=None, params=None): + """ + ``_ :arg job_id: The ID of the job to fetch - :arg snapshot_id: The ID of the snapshot to delete + :arg snapshot_id: The ID of the snapshot to revert to + :arg body: Reversion options + :arg delete_intervening_results: Should we reset the results back to the + time of the snapshot? """ for param in (job_id, snapshot_id): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'ml', 'anomaly_detectors', job_id, 'model_snapshots', snapshot_id), - params=params) + return self.transport.perform_request( + "POST", + _make_path( + "_ml", + "anomaly_detectors", + job_id, + "model_snapshots", + snapshot_id, + "_revert", + ), + params=params, + body=body, + ) + @query_params("enabled", "timeout") + def set_upgrade_mode(self, params=None): + """ + ``_ + + :arg enabled: Whether to enable upgrade_mode ML setting or not. Defaults + to false. + :arg timeout: Controls the time to wait before action times out. + Defaults to 30 seconds + """ + return self.transport.perform_request( + "POST", "/_ml/set_upgrade_mode", params=params + ) + + @query_params("end", "start", "timeout") + def start_datafeed(self, datafeed_id, body=None, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeed to start + :arg body: The start datafeed parameters + :arg end: The end time when the datafeed should stop. When not set, the + datafeed continues in real time + :arg start: The start time from where the datafeed should begin + :arg timeout: Controls the time to wait until a datafeed has started. + Default to 20 seconds + """ + if datafeed_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'datafeed_id'." + ) + return self.transport.perform_request( + "POST", + _make_path("_ml", "datafeeds", datafeed_id, "_start"), + params=params, + body=body, + ) + + @query_params("allow_no_datafeeds", "force", "timeout") + def stop_datafeed(self, datafeed_id, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeed to stop + :arg allow_no_datafeeds: Whether to ignore if a wildcard expression + matches no datafeeds. (This includes `_all` string or when no + datafeeds have been specified) + :arg force: True if the datafeed should be forcefully stopped. + :arg timeout: Controls the time to wait until a datafeed has stopped. + Default to 20 seconds + """ + if datafeed_id in SKIP_IN_PATH: + raise ValueError( + "Empty value passed for a required argument 'datafeed_id'." + ) + return self.transport.perform_request( + "POST", _make_path("_ml", "datafeeds", datafeed_id, "_stop"), params=params + ) + + @query_params() + def update_datafeed(self, datafeed_id, body, params=None): + """ + ``_ + + :arg datafeed_id: The ID of the datafeed to update + :arg body: The datafeed update settings + """ + for param in (datafeed_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "POST", + _make_path("_ml", "datafeeds", datafeed_id, "_update"), + params=params, + body=body, + ) + + @query_params() + def update_filter(self, filter_id, body, params=None): + """ + `<>`_ + + :arg filter_id: The ID of the filter to update + :arg body: The filter update + """ + for param in (filter_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "POST", + _make_path("_ml", "filters", filter_id, "_update"), + params=params, + body=body, + ) + + @query_params() + def update_job(self, job_id, body, params=None): + """ + ``_ + + :arg job_id: The ID of the job to create + :arg body: The job update settings + """ + for param in (job_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "POST", + _make_path("_ml", "anomaly_detectors", job_id, "_update"), + params=params, + body=body, + ) + + @query_params() + def update_model_snapshot(self, job_id, snapshot_id, body, params=None): + """ + ``_ + + :arg job_id: The ID of the job to fetch + :arg snapshot_id: The ID of the snapshot to update + :arg body: The model snapshot properties to update + """ + for param in (job_id, snapshot_id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "POST", + _make_path( + "_ml", + "anomaly_detectors", + job_id, + "model_snapshots", + snapshot_id, + "_update", + ), + params=params, + body=body, + ) + + @query_params() + def validate(self, body, params=None): + """ + `<>`_ + + :arg body: The job config + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "POST", "/_ml/anomaly_detectors/_validate", params=params, body=body + ) + + @query_params() + def validate_detector(self, body, params=None): + """ + `<>`_ + + :arg body: The detector + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "POST", + "/_ml/anomaly_detectors/_validate/detector", + params=params, + body=body, + ) diff --git a/elasticsearch/client/xpack/monitoring.py b/elasticsearch/client/xpack/monitoring.py index 5aca0ec5..b0776c0c 100644 --- a/elasticsearch/client/xpack/monitoring.py +++ b/elasticsearch/client/xpack/monitoring.py @@ -1,7 +1,8 @@ from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class MonitoringClient(NamespacedClient): - @query_params('interval', 'system_api_version', 'system_id') + @query_params("interval", "system_api_version", "system_id") def bulk(self, body, doc_type=None, params=None): """ ``_ @@ -16,6 +17,9 @@ class MonitoringClient(NamespacedClient): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('POST', _make_path('_xpack', - 'monitoring', doc_type, '_bulk'), params=params, - body=self.client._bulk_body(body)) + return self.transport.perform_request( + "POST", + _make_path("_monitoring", doc_type, "bulk"), + params=params, + body=self._bulk_body(body), + ) diff --git a/elasticsearch/client/xpack/rollup.py b/elasticsearch/client/xpack/rollup.py new file mode 100644 index 00000000..a2fe67da --- /dev/null +++ b/elasticsearch/client/xpack/rollup.py @@ -0,0 +1,124 @@ +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + + +class RollupClient(NamespacedClient): + @query_params() + def delete_job(self, id, params=None): + """ + `<>`_ + + :arg id: The ID of the job to delete + """ + if id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'id'.") + return self.transport.perform_request( + "DELETE", _make_path("_rollup", "job", id), params=params + ) + + @query_params() + def get_jobs(self, id=None, params=None): + """ + `<>`_ + + :arg id: The ID of the job(s) to fetch. Accepts glob patterns, or left + blank for all jobs + """ + return self.transport.perform_request( + "GET", _make_path("_rollup", "job", id), params=params + ) + + @query_params() + def get_rollup_caps(self, id=None, params=None): + """ + `<>`_ + + :arg id: The ID of the index to check rollup capabilities on, or left + blank for all jobs + """ + return self.transport.perform_request( + "GET", _make_path("_rollup", "data", id), params=params + ) + + @query_params() + def get_rollup_index_caps(self, index, params=None): + """ + `<>`_ + + :arg index: The rollup index or index pattern to obtain rollup + capabilities from. + """ + if index in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'index'.") + return self.transport.perform_request( + "GET", _make_path(index, "_rollup", "data"), params=params + ) + + @query_params() + def put_job(self, id, body, params=None): + """ + `<>`_ + + :arg id: The ID of the job to create + :arg body: The job configuration + """ + for param in (id, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", _make_path("_rollup", "job", id), params=params, body=body + ) + + @query_params("rest_total_hits_as_int", "typed_keys") + def rollup_search(self, index, body, doc_type=None, params=None): + """ + `<>`_ + + :arg index: The indices or index-pattern(s) (containing rollup or + regular data) that should be searched + :arg body: The search request body + :arg doc_type: The doc type inside the index + :arg rest_total_hits_as_int: Indicates whether hits.total should be + rendered as an integer or an object in the rest search response, + default False + :arg typed_keys: Specify whether aggregation and suggester names should + be prefixed by their respective types in the response + """ + for param in (index, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "GET", + _make_path(index, doc_type, "_rollup_search"), + params=params, + body=body, + ) + + @query_params() + def start_job(self, id, params=None): + """ + `<>`_ + + :arg id: The ID of the job to start + """ + if id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'id'.") + return self.transport.perform_request( + "POST", _make_path("_rollup", "job", id, "_start"), params=params + ) + + @query_params("timeout", "wait_for_completion") + def stop_job(self, id, params=None): + """ + `<>`_ + + :arg id: The ID of the job to stop + :arg timeout: Block for (at maximum) the specified duration while + waiting for the job to stop. Defaults to 30s. + :arg wait_for_completion: True if the API should block until the job has + fully stopped, false if should be executed async. Defaults to false. + """ + if id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'id'.") + return self.transport.perform_request( + "POST", _make_path("_rollup", "job", id, "_stop"), params=params + ) diff --git a/elasticsearch/client/xpack/security.py b/elasticsearch/client/xpack/security.py index 9a59bc39..81114fa4 100644 --- a/elasticsearch/client/xpack/security.py +++ b/elasticsearch/client/xpack/security.py @@ -1,103 +1,19 @@ from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class SecurityClient(NamespacedClient): - @query_params('refresh') - def delete_user(self, username, params=None): - """ - - ``_ - - :arg username: username - :arg refresh: If `true` (the default) then refresh the affected shards - to make this operation visible to search, if `wait_for` then wait - for a refresh to make this operation visible to search, if `false` - then do nothing with refreshes., valid choices are: 'true', 'false', - 'wait_for' - """ - if username in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'username'.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'security', 'user', username), params=params) - - @query_params() - def get_user(self, username=None, params=None): - """ - - ``_ - - :arg username: A comma-separated list of usernames - """ - return self.transport.perform_request('GET', _make_path('_xpack', - 'security', 'user', username), params=params) - - @query_params('refresh') - def put_role(self, name, body, params=None): - """ - - ``_ - - :arg name: Role name - :arg body: The role to add - :arg refresh: If `true` (the default) then refresh the affected shards - to make this operation visible to search, if `wait_for` then wait - for a refresh to make this operation visible to search, if `false` - then do nothing with refreshes., valid choices are: 'true', 'false', - 'wait_for' - """ - for param in (name, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'security', 'role', name), params=params, body=body) - @query_params() def authenticate(self, params=None): """ - ``_ """ - return self.transport.perform_request('GET', - '/_xpack/security/_authenticate', params=params) + return self.transport.perform_request( + "GET", "/_security/_authenticate", params=params + ) - @query_params('refresh') - def put_user(self, username, body, params=None): - """ - - ``_ - - :arg username: The username of the User - :arg body: The user to add - :arg refresh: If `true` (the default) then refresh the affected shards - to make this operation visible to search, if `wait_for` then wait - for a refresh to make this operation visible to search, if `false` - then do nothing with refreshes., valid choices are: 'true', 'false', - 'wait_for' - """ - for param in (username, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'security', 'user', username), params=params, body=body) - - @query_params('usernames') - def clear_cached_realms(self, realms, params=None): - """ - - ``_ - - :arg realms: Comma-separated list of realms to clear - :arg usernames: Comma-separated list of usernames to clear from the - cache - """ - if realms in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'realms'.") - return self.transport.perform_request('POST', _make_path('_xpack', - 'security', 'realm', realms, '_clear_cache'), params=params) - - @query_params('refresh') + @query_params("refresh") def change_password(self, body, username=None, params=None): """ - ``_ :arg body: the new password for the user @@ -110,39 +26,87 @@ class SecurityClient(NamespacedClient): """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'security', 'user', username, '_password'), params=params, - body=body) + return self.transport.perform_request( + "PUT", + _make_path("_security", "user", username, "_password"), + params=params, + body=body, + ) - @query_params() - def get_role(self, name=None, params=None): + @query_params("usernames") + def clear_cached_realms(self, realms, params=None): """ + ``_ - ``_ - - :arg name: Role name + :arg realms: Comma-separated list of realms to clear + :arg usernames: Comma-separated list of usernames to clear from the + cache """ - return self.transport.perform_request('GET', _make_path('_xpack', - 'security', 'role', name), params=params) + if realms in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'realms'.") + return self.transport.perform_request( + "POST", + _make_path("_security", "realm", realms, "_clear_cache"), + params=params, + ) @query_params() def clear_cached_roles(self, name, params=None): """ - - ``_ + ``_ :arg name: Role name """ if name in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'name'.") - return self.transport.perform_request('POST', _make_path('_xpack', - 'security', 'role', name, '_clear_cache'), params=params) + return self.transport.perform_request( + "POST", _make_path("_security", "role", name, "_clear_cache"), params=params + ) - @query_params('refresh') + @query_params("refresh") + def create_api_key(self, body, params=None): + """ + ``_ + + :arg body: The api key request to create an API key + :arg refresh: If `true` (the default) then refresh the affected shards + to make this operation visible to search, if `wait_for` then wait + for a refresh to make this operation visible to search, if `false` + then do nothing with refreshes., valid choices are: 'true', 'false', + 'wait_for' + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "PUT", "/_security/api_key", params=params, body=body + ) + + @query_params("refresh") + def delete_privileges(self, application, name, params=None): + """ + ``_ + + :arg application: Application name + :arg name: Privilege name + :arg refresh: If `true` (the default) then refresh the affected shards + to make this operation visible to search, if `wait_for` then wait + for a refresh to make this operation visible to search, if `false` + then do nothing with refreshes., valid choices are: 'true', 'false', + 'wait_for' + """ + for param in (application, name): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "DELETE", + _make_path("_security", "privilege", application, name), + params=params, + ) + + @query_params("refresh") def delete_role(self, name, params=None): """ - - ``_ + ``_ :arg name: Role name :arg refresh: If `true` (the default) then refresh the affected shards @@ -153,13 +117,14 @@ class SecurityClient(NamespacedClient): """ if name in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'name'.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'security', 'role', name), params=params) + return self.transport.perform_request( + "DELETE", _make_path("_security", "role", name), params=params + ) - @query_params('refresh') + @query_params("refresh") def delete_role_mapping(self, name, params=None): """ - ``_ + ``_ :arg name: Role-mapping name :arg refresh: If `true` (the default) then refresh the affected shards @@ -170,13 +135,32 @@ class SecurityClient(NamespacedClient): """ if name in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'name'.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'security', 'role_mapping', name), params=params) + return self.transport.perform_request( + "DELETE", _make_path("_security", "role_mapping", name), params=params + ) - @query_params('refresh') - def disable_user(self, username=None, params=None): + @query_params("refresh") + def delete_user(self, username, params=None): """ - ``_ + ``_ + + :arg username: username + :arg refresh: If `true` (the default) then refresh the affected shards + to make this operation visible to search, if `wait_for` then wait + for a refresh to make this operation visible to search, if `false` + then do nothing with refreshes., valid choices are: 'true', 'false', + 'wait_for' + """ + if username in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'username'.") + return self.transport.perform_request( + "DELETE", _make_path("_security", "user", username), params=params + ) + + @query_params("refresh") + def disable_user(self, username, params=None): + """ + ``_ :arg username: The username of the user to disable :arg refresh: If `true` (the default) then refresh the affected shards @@ -185,13 +169,16 @@ class SecurityClient(NamespacedClient): then do nothing with refreshes., valid choices are: 'true', 'false', 'wait_for' """ - return self.transport.perform_request('PUT', _make_path('_xpack', - 'security', 'user', username, '_disable'), params=params) + if username in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'username'.") + return self.transport.perform_request( + "PUT", _make_path("_security", "user", username, "_disable"), params=params + ) - @query_params('refresh') - def enable_user(self, username=None, params=None): + @query_params("refresh") + def enable_user(self, username, params=None): """ - ``_ + ``_ :arg username: The username of the user to enable :arg refresh: If `true` (the default) then refresh the affected shards @@ -200,47 +187,182 @@ class SecurityClient(NamespacedClient): then do nothing with refreshes., valid choices are: 'true', 'false', 'wait_for' """ - return self.transport.perform_request('PUT', _make_path('_xpack', - 'security', 'user', username, '_enable'), params=params) + if username in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'username'.") + return self.transport.perform_request( + "PUT", _make_path("_security", "user", username, "_enable"), params=params + ) + + @query_params("id", "name", "realm_name", "username") + def get_api_key(self, params=None): + """ + ``_ + + :arg id: API key id of the API key to be retrieved + :arg name: API key name of the API key to be retrieved + :arg realm_name: realm name of the user who created this API key to be + retrieved + :arg username: user name of the user who created this API key to be + retrieved + """ + return self.transport.perform_request( + "GET", "/_security/api_key", params=params + ) + + @query_params() + def get_privileges(self, application=None, name=None, params=None): + """ + ``_ + + :arg application: Application name + :arg name: Privilege name + """ + return self.transport.perform_request( + "GET", + _make_path("_security", "privilege", application, name), + params=params, + ) + + @query_params() + def get_role(self, name=None, params=None): + """ + ``_ + + :arg name: Role name + """ + return self.transport.perform_request( + "GET", _make_path("_security", "role", name), params=params + ) @query_params() def get_role_mapping(self, name=None, params=None): """ - ``_ + ``_ :arg name: Role-Mapping name """ - return self.transport.perform_request('GET', _make_path('_xpack', - 'security', 'role_mapping', name), params=params) + return self.transport.perform_request( + "GET", _make_path("_security", "role_mapping", name), params=params + ) @query_params() def get_token(self, body, params=None): """ - ``_ + ``_ :arg body: The token request to get """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('POST', - '/_xpack/security/oauth2/token', params=params, body=body) + return self.transport.perform_request( + "POST", "/_security/oauth2/token", params=params, body=body + ) + + @query_params() + def get_user(self, username=None, params=None): + """ + ``_ + + :arg username: A comma-separated list of usernames + """ + return self.transport.perform_request( + "GET", _make_path("_security", "user", username), params=params + ) + + @query_params() + def get_user_privileges(self, params=None): + """ + ``_ + """ + return self.transport.perform_request( + "GET", "/_security/user/_privileges", params=params + ) + + @query_params() + def has_privileges(self, body, user=None, params=None): + """ + ``_ + + :arg body: The privileges to test + :arg user: Username + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "GET", + _make_path("_security", "user", user, "_has_privileges"), + params=params, + body=body, + ) + + @query_params() + def invalidate_api_key(self, body, params=None): + """ + ``_ + + :arg body: The api key request to invalidate API key(s) + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "DELETE", "/_security/api_key", params=params, body=body + ) @query_params() def invalidate_token(self, body, params=None): """ - ``_ + ``_ :arg body: The token to invalidate """ if body in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'body'.") - return self.transport.perform_request('DELETE', - '/_xpack/security/oauth2/token', params=params, body=body) + return self.transport.perform_request( + "DELETE", "/_security/oauth2/token", params=params, body=body + ) - @query_params('refresh') + @query_params("refresh") + def put_privileges(self, body, params=None): + """ + ``_ + + :arg body: The privilege(s) to add + :arg refresh: If `true` (the default) then refresh the affected shards + to make this operation visible to search, if `wait_for` then wait + for a refresh to make this operation visible to search, if `false` + then do nothing with refreshes., valid choices are: 'true', 'false', + 'wait_for' + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "PUT", "/_security/privilege/", params=params, body=body + ) + + @query_params("refresh") + def put_role(self, name, body, params=None): + """ + ``_ + + :arg name: Role name + :arg body: The role to add + :arg refresh: If `true` (the default) then refresh the affected shards + to make this operation visible to search, if `wait_for` then wait + for a refresh to make this operation visible to search, if `false` + then do nothing with refreshes., valid choices are: 'true', 'false', + 'wait_for' + """ + for param in (name, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", _make_path("_security", "role", name), params=params, body=body + ) + + @query_params("refresh") def put_role_mapping(self, name, body, params=None): """ - ``_ + ``_ :arg name: Role-mapping name :arg body: The role to add @@ -253,6 +375,29 @@ class SecurityClient(NamespacedClient): for param in (name, body): if param in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'security', 'role_mapping', name), params=params, body=body) + return self.transport.perform_request( + "PUT", + _make_path("_security", "role_mapping", name), + params=params, + body=body, + ) + @query_params("refresh") + def put_user(self, username, body, params=None): + """ + ``_ + + :arg username: The username of the User + :arg body: The user to add + :arg refresh: If `true` (the default) then refresh the affected shards + to make this operation visible to search, if `wait_for` then wait + for a refresh to make this operation visible to search, if `false` + then do nothing with refreshes., valid choices are: 'true', 'false', + 'wait_for' + """ + for param in (username, body): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + return self.transport.perform_request( + "PUT", _make_path("_security", "user", username), params=params, body=body + ) diff --git a/elasticsearch/client/xpack/sql.py b/elasticsearch/client/xpack/sql.py new file mode 100644 index 00000000..cab2383f --- /dev/null +++ b/elasticsearch/client/xpack/sql.py @@ -0,0 +1,43 @@ +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + + +class SqlClient(NamespacedClient): + @query_params() + def clear_cursor(self, body, params=None): + """ + ``_ + + :arg body: Specify the cursor value in the `cursor` element to clean the + cursor. + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "POST", "/_sql/close", params=params, body=body + ) + + @query_params("format") + def query(self, body, params=None): + """ + ``_ + + :arg body: Use the `query` element to start a query. Use the `cursor` + element to continue a query. + :arg format: a short version of the Accept header, e.g. json, yaml + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request("POST", "/_sql", params=params, body=body) + + @query_params() + def translate(self, body, params=None): + """ + ``_ + + :arg body: Specify the query in the `query` element. + """ + if body in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'body'.") + return self.transport.perform_request( + "POST", "/_sql/translate", params=params, body=body + ) diff --git a/elasticsearch/client/xpack/ssl.py b/elasticsearch/client/xpack/ssl.py new file mode 100644 index 00000000..9043d679 --- /dev/null +++ b/elasticsearch/client/xpack/ssl.py @@ -0,0 +1,12 @@ +from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + + +class SslClient(NamespacedClient): + @query_params() + def certificates(self, params=None): + """ + ``_ + """ + return self.transport.perform_request( + "GET", "/_ssl/certificates", params=params + ) diff --git a/elasticsearch/client/xpack/watcher.py b/elasticsearch/client/xpack/watcher.py index 8e531019..657e1989 100644 --- a/elasticsearch/client/xpack/watcher.py +++ b/elasticsearch/client/xpack/watcher.py @@ -1,148 +1,136 @@ from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH + class WatcherClient(NamespacedClient): @query_params() - def stop(self, params=None): - """ - - ``_ - """ - return self.transport.perform_request('POST', '/_xpack/watcher/_stop', - params=params) - - @query_params('master_timeout') def ack_watch(self, watch_id, action_id=None, params=None): """ - ``_ :arg watch_id: Watch ID :arg action_id: A comma-separated list of the action ids to be acked - :arg master_timeout: Explicit operation timeout for connection to master - node """ if watch_id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'watch_id'.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'watcher', 'watch', watch_id, '_ack', action_id), params=params) + return self.transport.perform_request( + "PUT", + _make_path("_watcher", "watch", watch_id, "_ack", action_id), + params=params, + ) - @query_params('debug') + @query_params() + def activate_watch(self, watch_id, params=None): + """ + ``_ + + :arg watch_id: Watch ID + """ + if watch_id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'watch_id'.") + return self.transport.perform_request( + "PUT", _make_path("_watcher", "watch", watch_id, "_activate"), params=params + ) + + @query_params() + def deactivate_watch(self, watch_id, params=None): + """ + ``_ + + :arg watch_id: Watch ID + """ + if watch_id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'watch_id'.") + return self.transport.perform_request( + "PUT", + _make_path("_watcher", "watch", watch_id, "_deactivate"), + params=params, + ) + + @query_params() + def delete_watch(self, id, params=None): + """ + ``_ + + :arg id: Watch ID + """ + if id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'id'.") + return self.transport.perform_request( + "DELETE", _make_path("_watcher", "watch", id), params=params + ) + + @query_params("debug") def execute_watch(self, id=None, body=None, params=None): """ - ``_ :arg id: Watch ID :arg body: Execution control :arg debug: indicates whether the watch should execute in debug mode """ - return self.transport.perform_request('PUT', _make_path('_xpack', - 'watcher', 'watch', id, '_execute'), params=params, body=body) - - @query_params() - def start(self, params=None): - """ - - ``_ - """ - return self.transport.perform_request('POST', '/_xpack/watcher/_start', - params=params) - - @query_params('master_timeout') - def activate_watch(self, watch_id, params=None): - """ - - ``_ - - :arg watch_id: Watch ID - :arg master_timeout: Explicit operation timeout for connection to master - node - """ - if watch_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'watch_id'.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'watcher', 'watch', watch_id, '_activate'), params=params) - - @query_params('master_timeout') - def deactivate_watch(self, watch_id, params=None): - """ - - ``_ - - :arg watch_id: Watch ID - :arg master_timeout: Explicit operation timeout for connection to master - node - """ - if watch_id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'watch_id'.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'watcher', 'watch', watch_id, '_deactivate'), params=params) - - @query_params('active', 'master_timeout') - def put_watch(self, id, body, params=None): - """ - - ``_ - - :arg id: Watch ID - :arg body: The watch - :arg active: Specify whether the watch is in/active by default - :arg master_timeout: Explicit operation timeout for connection to master - node - """ - for param in (id, body): - if param in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument.") - return self.transport.perform_request('PUT', _make_path('_xpack', - 'watcher', 'watch', id), params=params, body=body) - - @query_params('master_timeout') - def delete_watch(self, id, params=None): - """ - - ``_ - - :arg id: Watch ID - :arg master_timeout: Explicit operation timeout for connection to master - node - """ - if id in SKIP_IN_PATH: - raise ValueError("Empty value passed for a required argument 'id'.") - return self.transport.perform_request('DELETE', _make_path('_xpack', - 'watcher', 'watch', id), params=params) + return self.transport.perform_request( + "PUT", + _make_path("_watcher", "watch", id, "_execute"), + params=params, + body=body, + ) @query_params() def get_watch(self, id, params=None): """ - ``_ :arg id: Watch ID """ if id in SKIP_IN_PATH: raise ValueError("Empty value passed for a required argument 'id'.") - return self.transport.perform_request('GET', _make_path('_xpack', - 'watcher', 'watch', id), params=params) + return self.transport.perform_request( + "GET", _make_path("_watcher", "watch", id), params=params + ) - @query_params('emit_stacktraces') + @query_params("active", "if_primary_term", "if_seq_no", "version") + def put_watch(self, id, body=None, params=None): + """ + ``_ + + :arg id: Watch ID + :arg body: The watch + :arg active: Specify whether the watch is in/active by default + :arg if_primary_term: only update the watch if the last operation that + has changed the watch has the specified primary term + :arg if_seq_no: only update the watch if the last operation that has + changed the watch has the specified sequence number + :arg version: Explicit version number for concurrency control + """ + if id in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument 'id'.") + return self.transport.perform_request( + "PUT", _make_path("_watcher", "watch", id), params=params, body=body + ) + + @query_params() + def start(self, params=None): + """ + ``_ + """ + return self.transport.perform_request("POST", "/_watcher/_start", params=params) + + @query_params("emit_stacktraces") def stats(self, metric=None, params=None): """ - ``_ :arg metric: Controls what additional stat metrics should be include in the response :arg emit_stacktraces: Emits stack traces of currently running watches """ - return self.transport.perform_request('GET', _make_path('_xpack', - 'watcher', 'stats', metric), params=params) + return self.transport.perform_request( + "GET", _make_path("_watcher", "stats", metric), params=params + ) @query_params() - def restart(self, params=None): + def stop(self, params=None): """ - - ``_ + ``_ """ - return self.transport.perform_request('POST', - '/_xpack/watcher/_restart', params=params) - + return self.transport.perform_request("POST", "/_watcher/_stop", params=params) diff --git a/elasticsearch/connection/http_urllib3.py b/elasticsearch/connection/http_urllib3.py index 90452fa2..d573b0b9 100644 --- a/elasticsearch/connection/http_urllib3.py +++ b/elasticsearch/connection/http_urllib3.py @@ -15,12 +15,18 @@ CA_CERTS = None try: import certifi + CA_CERTS = certifi.where() except ImportError: pass from .base import Connection -from ..exceptions import ConnectionError, ImproperlyConfigured, ConnectionTimeout, SSLError +from ..exceptions import ( + ConnectionError, + ImproperlyConfigured, + ConnectionTimeout, + SSLError, +) from ..compat import urlencode @@ -67,17 +73,36 @@ class Urllib3HttpConnection(Connection): :arg headers: any custom http headers to be add to requests :arg http_compress: Use gzip compression """ - def __init__(self, host='localhost', port=9200, http_auth=None, - use_ssl=False, verify_certs=VERIFY_CERTS_DEFAULT, ssl_show_warn=True, ca_certs=None, client_cert=None, - client_key=None, ssl_version=None, ssl_assert_hostname=None, - ssl_assert_fingerprint=None, maxsize=10, headers=None, ssl_context=None, http_compress=False, **kwargs): - super(Urllib3HttpConnection, self).__init__(host=host, port=port, use_ssl=use_ssl, **kwargs) + def __init__( + self, + host="localhost", + port=9200, + http_auth=None, + use_ssl=False, + verify_certs=VERIFY_CERTS_DEFAULT, + ssl_show_warn=True, + ca_certs=None, + client_cert=None, + client_key=None, + ssl_version=None, + ssl_assert_hostname=None, + ssl_assert_fingerprint=None, + maxsize=10, + headers=None, + ssl_context=None, + http_compress=False, + **kwargs + ): + + super(Urllib3HttpConnection, self).__init__( + host=host, port=port, use_ssl=use_ssl, **kwargs + ) self.http_compress = http_compress self.headers = urllib3.make_headers(keep_alive=True) if http_auth is not None: if isinstance(http_auth, (tuple, list)): - http_auth = ':'.join(http_auth) + http_auth = ":".join(http_auth) self.headers.update(urllib3.make_headers(basic_auth=http_auth)) # update headers in lowercase to allow overriding of auth headers @@ -87,32 +112,43 @@ class Urllib3HttpConnection(Connection): if self.http_compress == True: self.headers.update(urllib3.make_headers(accept_encoding=True)) - self.headers.update({'content-encoding': 'gzip'}) + self.headers.update({"content-encoding": "gzip"}) - self.headers.setdefault('content-type', 'application/json') + self.headers.setdefault("content-type", "application/json") pool_class = urllib3.HTTPConnectionPool kw = {} # if providing an SSL context, raise error if any other SSL related flag is used - if ssl_context and ( (verify_certs is not VERIFY_CERTS_DEFAULT) or ca_certs - or client_cert or client_key or ssl_version): - warnings.warn("When using `ssl_context`, all other SSL related kwargs are ignored") + if ssl_context and ( + (verify_certs is not VERIFY_CERTS_DEFAULT) + or ca_certs + or client_cert + or client_key + or ssl_version + ): + warnings.warn( + "When using `ssl_context`, all other SSL related kwargs are ignored" + ) # if ssl_context provided use SSL by default if ssl_context and self.use_ssl: pool_class = urllib3.HTTPSConnectionPool - kw.update({ - 'assert_fingerprint': ssl_assert_fingerprint, - 'ssl_context': ssl_context, - }) + kw.update( + { + "assert_fingerprint": ssl_assert_fingerprint, + "ssl_context": ssl_context, + } + ) elif self.use_ssl: pool_class = urllib3.HTTPSConnectionPool - kw.update({ - 'ssl_version': ssl_version, - 'assert_hostname': ssl_assert_hostname, - 'assert_fingerprint': ssl_assert_fingerprint, - }) + kw.update( + { + "ssl_version": ssl_version, + "assert_hostname": ssl_assert_hostname, + "assert_fingerprint": ssl_assert_fingerprint, + } + ) # If `verify_certs` is sentinal value, default `verify_certs` to `True` if verify_certs is VERIFY_CERTS_DEFAULT: @@ -121,43 +157,52 @@ class Urllib3HttpConnection(Connection): ca_certs = CA_CERTS if ca_certs is None else ca_certs if verify_certs: if not ca_certs: - raise ImproperlyConfigured("Root certificates are missing for certificate " + raise ImproperlyConfigured( + "Root certificates are missing for certificate " "validation. Either pass them in using the ca_certs parameter or " - "install certifi to use it automatically.") + "install certifi to use it automatically." + ) - kw.update({ - 'cert_reqs': 'CERT_REQUIRED', - 'ca_certs': ca_certs, - 'cert_file': client_cert, - 'key_file': client_key, - }) + kw.update( + { + "cert_reqs": "CERT_REQUIRED", + "ca_certs": ca_certs, + "cert_file": client_cert, + "key_file": client_key, + } + ) else: if ssl_show_warn: warnings.warn( - 'Connecting to %s using SSL with verify_certs=False is insecure.' % host) + "Connecting to %s using SSL with verify_certs=False is insecure." + % host + ) - self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize, **kw) + self.pool = pool_class( + host, port=port, timeout=self.timeout, maxsize=maxsize, **kw + ) - - def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None): + def perform_request( + self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None + ): url = self.url_prefix + url if params: - url = '%s?%s' % (url, urlencode(params)) + url = "%s?%s" % (url, urlencode(params)) full_url = self.host + url start = time.time() try: kw = {} if timeout: - kw['timeout'] = timeout + kw["timeout"] = timeout # in python2 we need to make sure the url and method are not # unicode. Otherwise the body will be decoded into unicode too and # that will fail (#133, #201). if not isinstance(url, str): - url = url.encode('utf-8') + url = url.encode("utf-8") if not isinstance(method, str): - method = method.encode('utf-8') + method = method.encode("utf-8") request_headers = self.headers if headers: @@ -171,24 +216,31 @@ class Urllib3HttpConnection(Connection): # again body = gzip.zlib.compress(body) - response = self.pool.urlopen(method, url, body, retries=Retry(False), headers=request_headers, **kw) + response = self.pool.urlopen( + method, url, body, retries=Retry(False), headers=request_headers, **kw + ) duration = time.time() - start - raw_data = response.data.decode('utf-8') + raw_data = response.data.decode("utf-8") except Exception as e: - self.log_request_fail(method, full_url, url, body, time.time() - start, exception=e) + self.log_request_fail( + method, full_url, url, body, time.time() - start, exception=e + ) if isinstance(e, UrllibSSLError): - raise SSLError('N/A', str(e), e) + raise SSLError("N/A", str(e), e) if isinstance(e, ReadTimeoutError): - raise ConnectionTimeout('TIMEOUT', str(e), e) - raise ConnectionError('N/A', str(e), e) + raise ConnectionTimeout("TIMEOUT", str(e), e) + raise ConnectionError("N/A", str(e), e) # raise errors based on http status codes, let the client handle those if needed if not (200 <= response.status < 300) and response.status not in ignore: - self.log_request_fail(method, full_url, url, body, duration, response.status, raw_data) + self.log_request_fail( + method, full_url, url, body, duration, response.status, raw_data + ) self._raise_error(response.status, raw_data) - self.log_request_success(method, full_url, url, body, response.status, - raw_data, duration) + self.log_request_success( + method, full_url, url, body, response.status, raw_data, duration + ) return response.status, response.getheaders(), raw_data diff --git a/elasticsearch/helpers/actions.py b/elasticsearch/helpers/actions.py index c13b313a..19dd1a89 100644 --- a/elasticsearch/helpers/actions.py +++ b/elasticsearch/helpers/actions.py @@ -1,5 +1,3 @@ - - from operator import methodcaller import time @@ -11,7 +9,7 @@ from .errors import ScanError, BulkIndexError import logging -logger = logging.getLogger('elasticsearch.helpers') +logger = logging.getLogger("elasticsearch.helpers") def expand_action(data): @@ -26,19 +24,33 @@ def expand_action(data): # make sure we don't alter the action data = data.copy() - op_type = data.pop('_op_type', 'index') + op_type = data.pop("_op_type", "index") action = {op_type: {}} - for key in ('_index', '_parent', '_percolate', '_routing', '_timestamp', 'routing', - '_type', '_version', '_version_type', '_id', - 'retry_on_conflict', 'pipeline'): + for key in ( + "_index", + "_parent", + "_percolate", + "_routing", + "_timestamp", + "routing", + "_type", + "_version", + "_version_type", + "_id", + "retry_on_conflict", + "pipeline", + ): if key in data: - action[op_type][key] = data.pop(key) + if key == "_routing": + action[op_type]["routing"] = data.pop(key) + else: + action[op_type][key] = data.pop(key) # no data payload for delete - if op_type == 'delete': + if op_type == "delete": return action, None - return action, data.get('_source', data) + return action, data.get("_source", data) def _chunk_actions(actions, chunk_size, max_chunk_bytes, serializer): @@ -52,14 +64,16 @@ def _chunk_actions(actions, chunk_size, max_chunk_bytes, serializer): raw_data, raw_action = data, action action = serializer.dumps(action) # +1 to account for the trailing new line character - cur_size = len(action.encode('utf-8')) + 1 + cur_size = len(action.encode("utf-8")) + 1 if data is not None: data = serializer.dumps(data) - cur_size += len(data.encode('utf-8')) + 1 + cur_size += len(data.encode("utf-8")) + 1 # full chunk, send it and start a new one - if bulk_actions and (size + cur_size > max_chunk_bytes or action_count == chunk_size): + if bulk_actions and ( + size + cur_size > max_chunk_bytes or action_count == chunk_size + ): yield bulk_data, bulk_actions bulk_actions, bulk_data = [], [] size, action_count = 0, 0 @@ -69,7 +83,7 @@ def _chunk_actions(actions, chunk_size, max_chunk_bytes, serializer): bulk_actions.append(data) bulk_data.append((raw_action, raw_data)) else: - bulk_data.append((raw_action, )) + bulk_data.append((raw_action,)) size += cur_size action_count += 1 @@ -78,7 +92,15 @@ def _chunk_actions(actions, chunk_size, max_chunk_bytes, serializer): yield bulk_data, bulk_actions -def _process_bulk_chunk(client, bulk_actions, bulk_data, raise_on_exception=True, raise_on_error=True, *args, **kwargs): +def _process_bulk_chunk( + client, + bulk_actions, + bulk_data, + raise_on_exception=True, + raise_on_error=True, + *args, + **kwargs +): """ Send a bulk request to elasticsearch and process the output. """ @@ -87,7 +109,7 @@ def _process_bulk_chunk(client, bulk_actions, bulk_data, raise_on_exception=True try: # send the actual request - resp = client.bulk('\n'.join(bulk_actions) + '\n', *args, **kwargs) + resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs) except TransportError as e: # default behavior - just propagate exception if raise_on_exception: @@ -101,26 +123,30 @@ def _process_bulk_chunk(client, bulk_actions, bulk_data, raise_on_exception=True # collect all the information about failed actions op_type, action = data[0].copy().popitem() info = {"error": err_message, "status": e.status_code, "exception": e} - if op_type != 'delete': - info['data'] = data[1] + if op_type != "delete": + info["data"] = data[1] info.update(action) exc_errors.append({op_type: info}) # emulate standard behavior for failed actions if raise_on_error: - raise BulkIndexError('%i document(s) failed to index.' % len(exc_errors), exc_errors) + raise BulkIndexError( + "%i document(s) failed to index." % len(exc_errors), exc_errors + ) else: for err in exc_errors: yield False, err return # go through request-response pairs and detect failures - for data, (op_type, item) in zip(bulk_data, map(methodcaller('popitem'), resp['items'])): - ok = 200 <= item.get('status', 500) < 300 + for data, (op_type, item) in zip( + bulk_data, map(methodcaller("popitem"), resp["items"]) + ): + ok = 200 <= item.get("status", 500) < 300 if not ok and raise_on_error: # include original document source if len(data) > 1: - item['data'] = data[1] + item["data"] = data[1] errors.append({op_type: item}) if ok or not errors: @@ -129,13 +155,24 @@ def _process_bulk_chunk(client, bulk_actions, bulk_data, raise_on_exception=True yield ok, {op_type: item} if errors: - raise BulkIndexError('%i document(s) failed to index.' % len(errors), errors) + raise BulkIndexError("%i document(s) failed to index." % len(errors), errors) -def streaming_bulk(client, actions, chunk_size=500, max_chunk_bytes=100 * 1024 * 1024, - raise_on_error=True, expand_action_callback=expand_action, - raise_on_exception=True, max_retries=0, initial_backoff=2, - max_backoff=600, yield_ok=True, *args, **kwargs): +def streaming_bulk( + client, + actions, + chunk_size=500, + max_chunk_bytes=100 * 1024 * 1024, + raise_on_error=True, + expand_action_callback=expand_action, + raise_on_exception=True, + max_retries=0, + initial_backoff=2, + max_backoff=600, + yield_ok=True, + *args, + **kwargs +): """ Streaming bulk consumes actions from the iterable passed in and yields @@ -171,33 +208,43 @@ def streaming_bulk(client, actions, chunk_size=500, max_chunk_bytes=100 * 1024 * """ actions = map(expand_action_callback, actions) - for bulk_data, bulk_actions in _chunk_actions(actions, chunk_size, - max_chunk_bytes, - client.transport.serializer): + for bulk_data, bulk_actions in _chunk_actions( + actions, chunk_size, max_chunk_bytes, client.transport.serializer + ): for attempt in range(max_retries + 1): to_retry, to_retry_data = [], [] if attempt: - time.sleep(min(max_backoff, initial_backoff * 2**(attempt-1))) + time.sleep(min(max_backoff, initial_backoff * 2 ** (attempt - 1))) try: for data, (ok, info) in zip( - bulk_data, - _process_bulk_chunk(client, bulk_actions, bulk_data, - raise_on_exception, - raise_on_error, *args, **kwargs) - ): + bulk_data, + _process_bulk_chunk( + client, + bulk_actions, + bulk_data, + raise_on_exception, + raise_on_error, + *args, + **kwargs + ), + ): if not ok: action, info = info.popitem() # retry if retries enabled, we get 429, and we are not # in the last attempt - if max_retries \ - and info['status'] == 429 \ - and (attempt+1) <= max_retries: + if ( + max_retries + and info["status"] == 429 + and (attempt + 1) <= max_retries + ): # _process_bulk_chunk expects strings so we need to # re-serialize the data - to_retry.extend(map(client.transport.serializer.dumps, data)) + to_retry.extend( + map(client.transport.serializer.dumps, data) + ) to_retry_data.append(data) else: yield ok, {action: info} @@ -249,7 +296,7 @@ def bulk(client, actions, stats_only=False, *args, **kwargs): errors = [] # make streaming_bulk yield successful results so we can count them - kwargs['yield_ok'] = True + kwargs["yield_ok"] = True for ok, item in streaming_bulk(client, actions, *args, **kwargs): # go through request-response pairs and detect failures if not ok: @@ -262,9 +309,17 @@ def bulk(client, actions, stats_only=False, *args, **kwargs): return success, failed if stats_only else errors -def parallel_bulk(client, actions, thread_count=4, chunk_size=500, - max_chunk_bytes=100 * 1024 * 1024, queue_size=4, - expand_action_callback=expand_action, *args, **kwargs): +def parallel_bulk( + client, + actions, + thread_count=4, + chunk_size=500, + max_chunk_bytes=100 * 1024 * 1024, + queue_size=4, + expand_action_callback=expand_action, + *args, + **kwargs +): """ Parallel version of the bulk helper run in multiple threads at once. @@ -299,9 +354,15 @@ def parallel_bulk(client, actions, thread_count=4, chunk_size=500, try: for result in pool.imap( - lambda bulk_chunk: list(_process_bulk_chunk(client, bulk_chunk[1], bulk_chunk[0], *args, **kwargs)), - _chunk_actions(actions, chunk_size, max_chunk_bytes, client.transport.serializer) - ): + lambda bulk_chunk: list( + _process_bulk_chunk( + client, bulk_chunk[1], bulk_chunk[0], *args, **kwargs + ) + ), + _chunk_actions( + actions, chunk_size, max_chunk_bytes, client.transport.serializer + ), + ): for item in result: yield item @@ -310,9 +371,18 @@ def parallel_bulk(client, actions, thread_count=4, chunk_size=500, pool.join() -def scan(client, query=None, scroll='5m', raise_on_error=True, - preserve_order=False, size=1000, request_timeout=None, clear_scroll=True, - scroll_kwargs=None, **kwargs): +def scan( + client, + query=None, + scroll="5m", + raise_on_error=True, + preserve_order=False, + size=1000, + request_timeout=None, + clear_scroll=True, + scroll_kwargs=None, + **kwargs +): """ Simple abstraction on top of the :meth:`~elasticsearch.Elasticsearch.scroll` api - a simple iterator that @@ -358,10 +428,11 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, query = query.copy() if query else {} query["sort"] = "_doc" # initial search - resp = client.search(body=query, scroll=scroll, size=size, - request_timeout=request_timeout, **kwargs) + resp = client.search( + body=query, scroll=scroll, size=size, request_timeout=request_timeout, **kwargs + ) - scroll_id = resp.get('_scroll_id') + scroll_id = resp.get("_scroll_id") if scroll_id is None: return @@ -372,37 +443,50 @@ def scan(client, query=None, scroll='5m', raise_on_error=True, if first_run: first_run = False else: - resp = client.scroll(scroll_id, scroll=scroll, - request_timeout=request_timeout, - **scroll_kwargs) + resp = client.scroll( + scroll_id, + scroll=scroll, + request_timeout=request_timeout, + **scroll_kwargs + ) - for hit in resp['hits']['hits']: + for hit in resp["hits"]["hits"]: yield hit # check if we have any errrors if resp["_shards"]["successful"] < resp["_shards"]["total"]: logger.warning( - 'Scroll request has only succeeded on %d shards out of %d.', - resp['_shards']['successful'], resp['_shards']['total'] + "Scroll request has only succeeded on %d shards out of %d.", + resp["_shards"]["successful"], + resp["_shards"]["total"], ) if raise_on_error: raise ScanError( scroll_id, - 'Scroll request has only succeeded on %d shards out of %d.' % - (resp['_shards']['successful'], resp['_shards']['total']) + "Scroll request has only succeeded on %d shards out of %d." + % (resp["_shards"]["successful"], resp["_shards"]["total"]), ) - scroll_id = resp.get('_scroll_id') + scroll_id = resp.get("_scroll_id") # end of scroll - if scroll_id is None or not resp['hits']['hits']: + if scroll_id is None or not resp["hits"]["hits"]: break finally: if scroll_id and clear_scroll: - client.clear_scroll(body={'scroll_id': [scroll_id]}, ignore=(404, )) + client.clear_scroll(body={"scroll_id": [scroll_id]}, ignore=(404,)) -def reindex(client, source_index, target_index, query=None, target_client=None, - chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}): +def reindex( + client, + source_index, + target_index, + query=None, + target_client=None, + chunk_size=500, + scroll="5m", + scan_kwargs={}, + bulk_kwargs={}, +): """ Reindex all documents from one index that satisfy a given query @@ -435,26 +519,20 @@ def reindex(client, source_index, target_index, query=None, target_client=None, :func:`~elasticsearch.helpers.bulk` """ target_client = client if target_client is None else target_client - - docs = scan(client, - query=query, - index=source_index, - scroll=scroll, - **scan_kwargs - ) + docs = scan(client, query=query, index=source_index, scroll=scroll, **scan_kwargs) def _change_doc_index(hits, index): for h in hits: - h['_index'] = index - if 'fields' in h: - h.update(h.pop('fields')) + h["_index"] = index + if "fields" in h: + h.update(h.pop("fields")) yield h - kwargs = { - 'stats_only': True, - } + kwargs = {"stats_only": True} kwargs.update(bulk_kwargs) - return bulk(target_client, - _change_doc_index(docs, target_index), - chunk_size=chunk_size, - **kwargs) + return bulk( + target_client, + _change_doc_index(docs, target_index), + chunk_size=chunk_size, + **kwargs + ) diff --git a/test_elasticsearch/run_tests.py b/test_elasticsearch/run_tests.py index 75e371fd..44928f9d 100755 --- a/test_elasticsearch/run_tests.py +++ b/test_elasticsearch/run_tests.py @@ -8,28 +8,31 @@ import subprocess import nose + def fetch_es_repo(): # user is manually setting YAML dir, don't tamper with it - if 'TEST_ES_YAML_DIR' in environ: + if "TEST_ES_YAML_DIR" in environ: return repo_path = environ.get( - 'TEST_ES_REPO', - abspath(join(dirname(__file__), pardir, pardir, 'elasticsearch')) + "TEST_ES_REPO", + abspath(join(dirname(__file__), pardir, pardir, "elasticsearch")), ) # no repo - if not exists(repo_path) or not exists(join(repo_path, '.git')): - print('No elasticsearch repo found...') + if not exists(repo_path) or not exists(join(repo_path, ".git")): + print("No elasticsearch repo found...") # set YAML DIR to empty to skip yaml tests - environ['TEST_ES_YAML_DIR'] = '' + environ["TEST_ES_YAML_DIR"] = "" return # set YAML test dir - environ['TEST_ES_YAML_DIR'] = join(repo_path, 'rest-api-spec', 'src', 'main', 'resources', 'rest-api-spec', 'test') + environ["TEST_ES_YAML_DIR"] = join( + repo_path, "rest-api-spec", "src", "main", "resources", "rest-api-spec", "test" + ) # fetching of yaml tests disabled, we'll run with what's there - if environ.get('TEST_ES_NOFETCH', False): + if environ.get("TEST_ES_NOFETCH", False): return from test_elasticsearch.test_server import get_client @@ -38,19 +41,25 @@ def fetch_es_repo(): # find out the sha of the running es try: es = get_client() - sha = es.info()['version']['build_hash'] + sha = es.info()["version"]["build_hash"] except (SkipTest, KeyError): - print('No running elasticsearch >1.X server...') + print("No running elasticsearch >1.X server...") return # fetch new commits to be sure... - print('Fetching elasticsearch repo...') - subprocess.check_call('cd %s && git fetch https://github.com/elasticsearch/elasticsearch.git' % repo_path, shell=True) + print("Fetching elasticsearch repo...") + subprocess.check_call( + "cd %s && git fetch https://github.com/elasticsearch/elasticsearch.git" + % repo_path, + shell=True, + ) # reset to the version fron info() - subprocess.check_call('cd %s && git reset --hard %s' % (repo_path, sha), shell=True) + subprocess.check_call("cd %s && git pull" % repo_path, shell=True) + subprocess.check_call("cd %s && git reset --hard %s" % (repo_path, sha), shell=True) + def run_all(argv=None): - sys.exitfunc = lambda: sys.stderr.write('Shutting down....\n') + sys.exitfunc = lambda: sys.stderr.write("Shutting down....\n") # fetch yaml tests fetch_es_repo() @@ -58,16 +67,19 @@ def run_all(argv=None): # always insert coverage when running tests if argv is None: argv = [ - 'nosetests', '--with-xunit', - '--with-xcoverage', '--cover-package=elasticsearch', '--cover-erase', - '--logging-filter=elasticsearch', '--logging-level=DEBUG', - '--verbose', '--with-id', + "nosetests", + "--with-xunit", + "--with-xcoverage", + "--cover-package=elasticsearch", + "--cover-erase", + "--logging-filter=elasticsearch", + "--logging-level=DEBUG", + "--verbose", + "--with-id", ] - nose.run_exit( - argv=argv, - defaultTest=abspath(dirname(__file__)) - ) + nose.run_exit(argv=argv, defaultTest=abspath(dirname(__file__))) -if __name__ == '__main__': + +if __name__ == "__main__": run_all(sys.argv) diff --git a/test_elasticsearch/start_elasticsearch.sh b/test_elasticsearch/start_elasticsearch.sh index e89d3473..9afa2f61 100755 --- a/test_elasticsearch/start_elasticsearch.sh +++ b/test_elasticsearch/start_elasticsearch.sh @@ -2,7 +2,7 @@ # Start elasticsearch in a docker container -ES_VERSION=${ES_VERSION:-"6.1.3"} +ES_VERSION=${ES_VERSION:-"7.0.0-beta1"} ES_TEST_SERVER=${ES_TEST_SERVER:-"http://localhost:9200"} SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -11,7 +11,9 @@ exec docker run -d \ -e path.repo=/tmp \ -e "repositories.url.allowed_urls=http://*" \ -e node.attr.testattr=test \ + -e node.name=test \ -e ES_HOST=$ES_TEST_SERVER \ + -e cluster.initial_master_nodes=test \ -v $SOURCE_DIR/../elasticsearch:/code/elasticsearch \ -v /tmp:/tmp \ -p "9200:9200" \ diff --git a/test_elasticsearch/test_client/__init__.py b/test_elasticsearch/test_client/__init__.py index 25079663..8e0c67ca 100644 --- a/test_elasticsearch/test_client/__init__.py +++ b/test_elasticsearch/test_client/__init__.py @@ -4,6 +4,7 @@ from elasticsearch.client import _normalize_hosts, Elasticsearch from ..test_cases import TestCase, ElasticsearchTestCase + class TestNormalizeHosts(TestCase): def test_none_uses_defaults(self): self.assertEquals([{}], _normalize_hosts(None)) @@ -13,87 +14,86 @@ class TestNormalizeHosts(TestCase): def test_strings_are_parsed_for_port_and_user(self): self.assertEquals( - [{"host": "elastic.co", "port": 42}, {"host": "elastic.co", "http_auth": "user:secre]"}], - _normalize_hosts(["elastic.co:42", "user:secre%5D@elastic.co"]) + [ + {"host": "elastic.co", "port": 42}, + {"host": "elastic.co", "http_auth": "user:secre]"}, + ], + _normalize_hosts(["elastic.co:42", "user:secre%5D@elastic.co"]), ) def test_strings_are_parsed_for_scheme(self): self.assertEquals( [ - { - "host": "elastic.co", - "port": 42, - "use_ssl": True, - }, + {"host": "elastic.co", "port": 42, "use_ssl": True}, { "host": "elastic.co", "http_auth": "user:secret", "use_ssl": True, "port": 443, - 'url_prefix': '/prefix' - } + "url_prefix": "/prefix", + }, ], - _normalize_hosts(["https://elastic.co:42", "https://user:secret@elastic.co/prefix"]) + _normalize_hosts( + ["https://elastic.co:42", "https://user:secret@elastic.co/prefix"] + ), ) def test_dicts_are_left_unchanged(self): - self.assertEquals([{"host": "local", "extra": 123}], _normalize_hosts([{"host": "local", "extra": 123}])) + self.assertEquals( + [{"host": "local", "extra": 123}], + _normalize_hosts([{"host": "local", "extra": 123}]), + ) def test_single_string_is_wrapped_in_list(self): - self.assertEquals( - [{"host": "elastic.co"}], - _normalize_hosts("elastic.co") - ) + self.assertEquals([{"host": "elastic.co"}], _normalize_hosts("elastic.co")) class TestClient(ElasticsearchTestCase): def test_request_timeout_is_passed_through_unescaped(self): - self.client.ping(request_timeout=.1) - calls = self.assert_url_called('HEAD', '/') - self.assertEquals([({'request_timeout': .1}, None)], calls) + self.client.ping(request_timeout=0.1) + calls = self.assert_url_called("HEAD", "/") + self.assertEquals([({"request_timeout": 0.1}, None)], calls) def test_params_is_copied_when(self): rt = object() params = dict(request_timeout=rt) self.client.ping(params=params) self.client.ping(params=params) - calls = self.assert_url_called('HEAD', '/', 2) + calls = self.assert_url_called("HEAD", "/", 2) self.assertEquals( - [ - ({'request_timeout': rt}, None), - ({'request_timeout': rt}, None) - ], - calls + [({"request_timeout": rt}, None), ({"request_timeout": rt}, None)], calls ) self.assertFalse(calls[0][0] is calls[1][0]) def test_from_in_search(self): - self.client.search(index='i', doc_type='t', from_=10) - calls = self.assert_url_called('GET', '/i/t/_search') - self.assertEquals([({'from': '10'}, None)], calls) + self.client.search(index="i", from_=10) + calls = self.assert_url_called("GET", "/i/_search") + self.assertEquals([({"from": "10"}, None)], calls) def test_repr_contains_hosts(self): - self.assertEquals('', repr(self.client)) + self.assertEquals("", repr(self.client)) def test_repr_subclass(self): - class OtherElasticsearch(Elasticsearch): pass - self.assertEqual('', repr(OtherElasticsearch())) + class OtherElasticsearch(Elasticsearch): + pass + + self.assertEqual("", repr(OtherElasticsearch())) def test_repr_contains_hosts_passed_in(self): - self.assertIn("es.org", repr(Elasticsearch(['es.org:123']))) + self.assertIn("es.org", repr(Elasticsearch(["es.org:123"]))) def test_repr_truncates_host_to_5(self): hosts = [{"host": "es" + str(i)} for i in range(10)] es = Elasticsearch(hosts) self.assertNotIn("es5", repr(es)) - self.assertIn('...', repr(es)) + self.assertIn("...", repr(es)) def test_index_uses_post_if_id_is_empty(self): - self.client.index(index='my-index', doc_type='test-doc', id='', body={}) + self.client.index(index="my-index", id="", body={}) - self.assert_url_called('POST', '/my-index/test-doc') + self.assert_url_called("POST", "/my-index/_doc") def test_index_uses_put_if_id_is_not_empty(self): - self.client.index(index='my-index', doc_type='test-doc', id=0, body={}) + self.client.index(index="my-index", id=0, body={}) - self.assert_url_called('PUT', '/my-index/test-doc/0') + self.assert_url_called("PUT", "/my-index/_doc/0") diff --git a/test_elasticsearch/test_helpers.py b/test_elasticsearch/test_helpers.py index f73a4f09..b95792f7 100644 --- a/test_elasticsearch/test_helpers.py +++ b/test_elasticsearch/test_helpers.py @@ -2,7 +2,7 @@ import mock import time import threading - +from nose.plugins.skip import SkipTest from elasticsearch import helpers, Elasticsearch from elasticsearch.serializer import JSONSerializer @@ -10,6 +10,7 @@ from .test_cases import TestCase lock_side_effect = threading.Lock() + def mock_process_bulk_chunk(*args, **kwargs): """ Threadsafe way of mocking process bulk chunk: @@ -26,43 +27,73 @@ mock_process_bulk_chunk.call_count = 0 class TestParallelBulk(TestCase): - @mock.patch('elasticsearch.helpers.actions._process_bulk_chunk', side_effect=mock_process_bulk_chunk) + @mock.patch( + "elasticsearch.helpers.actions._process_bulk_chunk", + side_effect=mock_process_bulk_chunk, + ) def test_all_chunks_sent(self, _process_bulk_chunk): - actions = ({'x': i} for i in range(100)) + actions = ({"x": i} for i in range(100)) list(helpers.parallel_bulk(Elasticsearch(), actions, chunk_size=2)) self.assertEquals(50, _process_bulk_chunk.call_count) + @SkipTest @mock.patch( - 'elasticsearch.helpers.actions._process_bulk_chunk', + "elasticsearch.helpers.actions._process_bulk_chunk", # make sure we spend some time in the thread - side_effect=lambda *a: [(True, time.sleep(.001) or threading.current_thread().ident)] + side_effect=lambda *a: [ + (True, time.sleep(0.001) or threading.current_thread().ident) + ], ) def test_chunk_sent_from_different_threads(self, _process_bulk_chunk): - actions = ({'x': i} for i in range(100)) - results = list(helpers.parallel_bulk(Elasticsearch(), actions, thread_count=10, chunk_size=2)) + actions = ({"x": i} for i in range(100)) + results = list( + helpers.parallel_bulk( + Elasticsearch(), actions, thread_count=10, chunk_size=2 + ) + ) self.assertTrue(len(set([r[1] for r in results])) > 1) + class TestChunkActions(TestCase): def setUp(self): super(TestChunkActions, self).setUp() - self.actions = [({'index': {}}, {'some': u'datá', 'i': i}) for i in range(100)] + self.actions = [({"index": {}}, {"some": u"datá", "i": i}) for i in range(100)] def test_chunks_are_chopped_by_byte_size(self): - self.assertEquals(100, len(list(helpers._chunk_actions(self.actions, 100000, 1, JSONSerializer())))) + self.assertEquals( + 100, + len( + list(helpers._chunk_actions(self.actions, 100000, 1, JSONSerializer())) + ), + ) def test_chunks_are_chopped_by_chunk_size(self): - self.assertEquals(10, len(list(helpers._chunk_actions(self.actions, 10, 99999999, JSONSerializer())))) + self.assertEquals( + 10, + len( + list( + helpers._chunk_actions(self.actions, 10, 99999999, JSONSerializer()) + ) + ), + ) def test_chunks_are_chopped_by_byte_size_properly(self): max_byte_size = 170 - chunks = list(helpers._chunk_actions(self.actions, 100000, max_byte_size, JSONSerializer())) + chunks = list( + helpers._chunk_actions( + self.actions, 100000, max_byte_size, JSONSerializer() + ) + ) self.assertEquals(25, len(chunks)) for chunk_data, chunk_actions in chunks: - chunk = u''.join(chunk_actions) - chunk = chunk if isinstance(chunk, str) else chunk.encode('utf-8') + chunk = u"".join(chunk_actions) + chunk = chunk if isinstance(chunk, str) else chunk.encode("utf-8") self.assertLessEqual(len(chunk), max_byte_size) + class TestExpandActions(TestCase): def test_string_actions_are_marked_as_simple_inserts(self): - self.assertEquals(('{"index":{}}', "whatever"), helpers.expand_action('whatever')) + self.assertEquals( + ('{"index":{}}', "whatever"), helpers.expand_action("whatever") + ) diff --git a/test_elasticsearch/test_server/test_common.py b/test_elasticsearch/test_server/test_common.py index 6261527f..5c9d477a 100644 --- a/test_elasticsearch/test_server/test_common.py +++ b/test_elasticsearch/test_server/test_common.py @@ -18,59 +18,66 @@ from . import ElasticsearchTestCase # some params had to be changed in python, keep track of them so we can rename # those in the tests accordingly -PARAMS_RENAMES = { - 'type': 'doc_type', - 'from': 'from_', -} +PARAMS_RENAMES = {"type": "doc_type", "from": "from_"} # mapping from catch values to http status codes -CATCH_CODES = { - 'missing': 404, - 'conflict': 409, - 'unauthorized': 401, -} +CATCH_CODES = {"missing": 404, "conflict": 409, "unauthorized": 401} # test features we have implemented -IMPLEMENTED_FEATURES = ('gtelte', 'stash_in_path', 'headers', 'catch_unauthorized') +IMPLEMENTED_FEATURES = ("gtelte", "stash_in_path", "headers", "catch_unauthorized") # broken YAML tests on some releases SKIP_TESTS = { # wait for https://github.com/elastic/elasticsearch/issues/25694 to be fixed # upstream - '*': set(('TestBulk10Basic', 'TestCatSnapshots10Basic', - 'TestClusterPutSettings10Basic', 'TestIndex10WithId', - 'TestClusterPutScript10Basic', 'TestIndicesPutMapping10Basic', - 'TestIndicesPutTemplate10Basic', 'TestMsearch10Basic', - # Skipping some broken integration tests: - 'TestIndicesOpen10Basic', 'TestClusterRemoteInfo10Info', - 'TestIndicesSplit10Basic', 'TestIndicesSplit20SourceMapping' - )) - + "*": set( + ( + "TestBulk10Basic", + "TestCatSnapshots10Basic", + "TestClusterPutSettings10Basic", + "TestIndex10WithId", + "TestClusterPutScript10Basic", + "TestIndicesPutMapping10Basic", + "TestIndicesPutTemplate10Basic", + "TestMsearch10Basic", + # Skipping some broken integration tests: + "TestIndicesOpen10Basic", + "TestClusterRemoteInfo10Info", + "TestIndicesSplit10Basic", + "TestIndicesSplit20SourceMapping", + ) + ) } + class InvalidActionType(Exception): pass + class YamlTestCase(ElasticsearchTestCase): def setUp(self): super(YamlTestCase, self).setUp() - if hasattr(self, '_setup_code'): + if hasattr(self, "_setup_code"): self.run_code(self._setup_code) self.last_response = None self._state = {} def tearDown(self): - if hasattr(self, '_teardown_code'): + if hasattr(self, "_teardown_code"): self.run_code(self._teardown_code) super(YamlTestCase, self).tearDown() - for repo, definition in self.client.snapshot.get_repository(repository='_all').items(): + for repo, definition in self.client.snapshot.get_repository( + repository="_all" + ).items(): self.client.snapshot.delete_repository(repository=repo) - if definition['type'] == 'fs': - rmtree('/tmp/%s' % definition['settings']['location'], ignore_errors=True) + if definition["type"] == "fs": + rmtree( + "/tmp/%s" % definition["settings"]["location"], ignore_errors=True + ) def _resolve(self, value): # resolve variables - if isinstance(value, string_types) and value.startswith('$'): + if isinstance(value, string_types) and value.startswith("$"): value = value[1:] self.assertIn(value, self._state) value = self._state[value] @@ -85,13 +92,13 @@ class YamlTestCase(ElasticsearchTestCase): def _lookup(self, path): # fetch the possibly nested value from last_response value = self.last_response - if path == '$body': + if path == "$body": return value - path = path.replace(r'\.', '\1') - for step in path.split('.'): + path = path.replace(r"\.", "\1") + for step in path.split("."): if not step: continue - step = step.replace('\1', '.') + step = step.replace("\1", ".") step = self._resolve(step) if step.isdigit() and step not in value: step = int(step) @@ -108,24 +115,24 @@ class YamlTestCase(ElasticsearchTestCase): self.assertEquals(1, len(action)) action_type, action = list(action.items())[0] - if hasattr(self, 'run_' + action_type): - getattr(self, 'run_' + action_type)(action) + if hasattr(self, "run_" + action_type): + getattr(self, "run_" + action_type)(action) else: raise InvalidActionType(action_type) def run_do(self, action): """ Perform an api call with given parameters. """ api = self.client - if 'headers' in action: - api = self._get_client(headers=action.pop('headers')) + if "headers" in action: + api = self._get_client(headers=action.pop("headers")) - catch = action.pop('catch', None) + catch = action.pop("catch", None) self.assertEquals(1, len(action)) method, args = list(action.items())[0] # locate api endpoint - for m in method.split('.'): + for m in method.split("."): self.assertTrue(hasattr(api, m)) api = getattr(api, m) @@ -147,55 +154,76 @@ class YamlTestCase(ElasticsearchTestCase): self.run_catch(catch, e) else: if catch: - raise AssertionError('Failed to catch %r in %r.' % (catch, self.last_response)) + raise AssertionError( + "Failed to catch %r in %r." % (catch, self.last_response) + ) def _get_nodes(self): - if not hasattr(self, '_node_info'): - self._node_info = list(self.client.nodes.info(node_id='_all', metric='clear')['nodes'].values()) + if not hasattr(self, "_node_info"): + self._node_info = list( + self.client.nodes.info(node_id="_all", metric="clear")["nodes"].values() + ) return self._node_info def _get_data_nodes(self): - return len([info for info in self._get_nodes() if info.get('attributes', {}).get('data', 'true') == 'true']) + return len( + [ + info + for info in self._get_nodes() + if info.get("attributes", {}).get("data", "true") == "true" + ] + ) def _get_benchmark_nodes(self): - return len([info for info in self._get_nodes() if info.get('attributes', {}).get('bench', 'false') == 'true']) + return len( + [ + info + for info in self._get_nodes() + if info.get("attributes", {}).get("bench", "false") == "true" + ] + ) def run_skip(self, skip): - if 'features' in skip: - features = skip['features'] + if "features" in skip: + features = skip["features"] if not isinstance(features, (tuple, list)): features = [features] for feature in features: if feature in IMPLEMENTED_FEATURES: continue - elif feature == 'requires_replica': + elif feature == "requires_replica": if self._get_data_nodes() > 1: continue - elif feature == 'benchmark': + elif feature == "benchmark": if self._get_benchmark_nodes(): continue - raise SkipTest(skip.get('reason', 'Feature %s is not supported' % feature)) + raise SkipTest( + skip.get("reason", "Feature %s is not supported" % feature) + ) - if 'version' in skip: - version, reason = skip['version'], skip['reason'] - if version == 'all': + if "version" in skip: + version, reason = skip["version"], skip["reason"] + if version == "all": raise SkipTest(reason) - min_version, max_version = version.split('-') - min_version = _get_version(min_version) or (0, ) - max_version = _get_version(max_version) or (999, ) - if min_version <= self.es_version <= max_version: + min_version, max_version = version.split("-") + min_version = _get_version(min_version) or (0,) + max_version = _get_version(max_version) or (999,) + if min_version <= self.es_version <= max_version: raise SkipTest(reason) def run_catch(self, catch, exception): - if catch == 'param': + if catch == "param": self.assertIsInstance(exception, TypeError) return self.assertIsInstance(exception, TransportError) if catch in CATCH_CODES: self.assertEquals(CATCH_CODES[catch], exception.status_code) - elif catch[0] == '/' and catch[-1] == '/': - self.assertTrue(re.search(catch[1:-1], exception.error + ' ' + repr(exception.info)), '%s not in %r' % (catch, exception.info)) + elif catch[0] == "/" and catch[-1] == "/": + self.assertTrue( + re.search(catch[1:-1], exception.error + " " + repr(exception.info)), + "%s not in %r" % (catch, exception.info), + ) self.last_response = exception.info def run_gt(self, action): @@ -229,11 +257,11 @@ class YamlTestCase(ElasticsearchTestCase): except AssertionError: pass else: - self.assertIn(value, ('', None, False, 0)) + self.assertIn(value, ("", None, False, 0)) def run_is_true(self, action): value = self._lookup(action) - self.assertNotIn(value, ('', None, False, 0)) + self.assertNotIn(value, ("", None, False, 0)) def run_length(self, action): for path, expected in action.items(): @@ -246,8 +274,11 @@ class YamlTestCase(ElasticsearchTestCase): value = self._lookup(path) expected = self._resolve(expected) - if isinstance(expected, string_types) and \ - expected.startswith('/') and expected.endswith('/'): + if ( + isinstance(expected, string_types) + and expected.startswith("/") + and expected.endswith("/") + ): expected = re.compile(expected[1:-1], re.VERBOSE) self.assertTrue(expected.search(value)) else: @@ -259,51 +290,75 @@ def construct_case(filename, name): Parse a definition of a test case from a yaml file and construct the TestCase subclass dynamically. """ + def make_test(test_name, definition, i): def m(self): - if name in SKIP_TESTS.get(self.es_version, ()) or name in SKIP_TESTS.get('*', ()): + if name in SKIP_TESTS.get(self.es_version, ()) or name in SKIP_TESTS.get( + "*", () + ): raise SkipTest() self.run_code(definition) - m.__doc__ = '%s:%s.test_from_yaml_%d (%s): %s' % ( - __name__, name, i, '/'.join(filename.split('/')[-2:]), test_name) - m.__name__ = 'test_from_yaml_%d' % i + + m.__doc__ = "%s:%s.test_from_yaml_%d (%s): %s" % ( + __name__, + name, + i, + "/".join(filename.split("/")[-2:]), + test_name, + ) + m.__name__ = "test_from_yaml_%d" % i return m with open(filename) as f: tests = list(yaml.load_all(f)) - attrs = { - '_yaml_file': filename - } + attrs = {"_yaml_file": filename} i = 0 for test in tests: for test_name, definition in test.items(): - if test_name in ('setup', 'teardown'): - attrs['_%s_code' % test_name] = definition + if test_name in ("setup", "teardown"): + attrs["_%s_code" % test_name] = definition continue - attrs['test_from_yaml_%d' % i] = make_test(test_name, definition, i) + attrs["test_from_yaml_%d" % i] = make_test(test_name, definition, i) i += 1 - return type(name, (YamlTestCase, ), attrs) + return type(name, (YamlTestCase,), attrs) + YAML_DIR = environ.get( - 'TEST_ES_YAML_DIR', + "TEST_ES_YAML_DIR", join( - dirname(__file__), pardir, pardir, pardir, - 'elasticsearch', 'rest-api-spec', 'src', 'main', 'resources', 'rest-api-spec', 'test' - ) + dirname(__file__), + pardir, + pardir, + pardir, + "elasticsearch", + "rest-api-spec", + "src", + "main", + "resources", + "rest-api-spec", + "test", + ), ) if exists(YAML_DIR): -# find all the test definitions in yaml files ... + # find all the test definitions in yaml files ... for (path, dirs, files) in walk(YAML_DIR): for filename in files: - if not filename.endswith(('.yaml', '.yml')): + if not filename.endswith((".yaml", ".yml")): continue # ... parse them - name = ('Test' + ''.join(s.title() for s in path[len(YAML_DIR) + 1:].split('/')) + filename.rsplit('.', 1)[0].title()).replace('_', '').replace('.', '') + name = ( + ( + "Test" + + "".join(s.title() for s in path[len(YAML_DIR) + 1 :].split("/")) + + filename.rsplit(".", 1)[0].title() + ) + .replace("_", "") + .replace(".", "") + ) # and insert them into locals for test runner to find them locals()[name] = construct_case(join(path, filename), name) - diff --git a/test_elasticsearch/test_server/test_helpers.py b/test_elasticsearch/test_server/test_helpers.py index 2190a05f..1e307d51 100644 --- a/test_elasticsearch/test_server/test_helpers.py +++ b/test_elasticsearch/test_server/test_helpers.py @@ -5,7 +5,9 @@ from ..test_cases import SkipTest class FailingBulkClient(object): - def __init__(self, client, fail_at=(2, ), fail_with=TransportError(599, "Error!", {})): + def __init__( + self, client, fail_at=(2,), fail_with=TransportError(599, "Error!", {}) + ): self.client = client self._called = 0 self._fail_at = fail_at @@ -16,34 +18,44 @@ class FailingBulkClient(object): self._called += 1 if self._called in self._fail_at: raise self._fail_with - return self.client.bulk(*args, **kwargs) + return self.client.bulk(*args, **kwargs) + class TestStreamingBulk(ElasticsearchTestCase): def test_actions_remain_unchanged(self): - actions = [{'_id': 1}, {'_id': 2}] - for ok, item in helpers.streaming_bulk(self.client, actions, index='test-index', doc_type='answers'): + actions = [{"_id": 1}, {"_id": 2}] + for ok, item in helpers.streaming_bulk( + self.client, actions, index="test-index" + ): self.assertTrue(ok) - self.assertEquals([{'_id': 1}, {'_id': 2}], actions) + self.assertEquals([{"_id": 1}, {"_id": 2}], actions) def test_all_documents_get_inserted(self): - docs = [{"answer": x, '_id': x} for x in range(100)] - for ok, item in helpers.streaming_bulk(self.client, docs, index='test-index', doc_type='answers', refresh=True): + docs = [{"answer": x, "_id": x} for x in range(100)] + for ok, item in helpers.streaming_bulk( + self.client, docs, index="test-index", refresh=True + ): self.assertTrue(ok) - self.assertEquals(100, self.client.count(index='test-index', doc_type='answers')['count']) - self.assertEquals({"answer": 42}, self.client.get(index='test-index', doc_type='answers', id=42)['_source']) + self.assertEquals(100, self.client.count(index="test-index")["count"]) + self.assertEquals( + {"answer": 42}, self.client.get(index="test-index", id=42)["_source"] + ) def test_all_errors_from_chunk_are_raised_on_failure(self): - self.client.indices.create("i", + self.client.indices.create( + "i", { - "mappings": {"t": {"properties": {"a": {"type": "integer"}}}}, - "settings": {"number_of_shards": 1, "number_of_replicas": 0} - }) + "mappings": {"properties": {"a": {"type": "integer"}}}, + "settings": {"number_of_shards": 1, "number_of_replicas": 0}, + }, + ) self.client.cluster.health(wait_for_status="yellow") try: - for ok, item in helpers.streaming_bulk(self.client, [{"a": "b"}, - {"a": "c"}], index="i", doc_type="t", raise_on_error=True): + for ok, item in helpers.streaming_bulk( + self.client, [{"a": "b"}, {"a": "c"}], index="i", raise_on_error=True + ): self.assertTrue(ok) except helpers.BulkIndexError as e: self.assertEquals(2, len(e.errors)) @@ -52,102 +64,136 @@ class TestStreamingBulk(ElasticsearchTestCase): def test_different_op_types(self): if self.es_version < (0, 90, 1): - raise SkipTest('update supported since 0.90.1') - self.client.index(index='i', doc_type='t', id=45, body={}) - self.client.index(index='i', doc_type='t', id=42, body={}) + raise SkipTest("update supported since 0.90.1") + self.client.index(index="i", id=45, body={}) + self.client.index(index="i", id=42, body={}) docs = [ - {'_index': 'i', '_type': 't', '_id': 47, 'f': 'v'}, - {'_op_type': 'delete', '_index': 'i', '_type': 't', '_id': 45}, - {'_op_type': 'update', '_index': 'i', '_type': 't', '_id': 42, 'doc': {'answer': 42}} + {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"}, + {"_op_type": "delete", "_index": "i", "_type": "_doc", "_id": 45}, + { + "_op_type": "update", + "_index": "i", + "_type": "_doc", + "_id": 42, + "doc": {"answer": 42}, + }, ] for ok, item in helpers.streaming_bulk(self.client, docs): self.assertTrue(ok) - self.assertFalse(self.client.exists(index='i', doc_type='t', id=45)) - self.assertEquals({'answer': 42}, self.client.get(index='i', doc_type='t', id=42)['_source']) - self.assertEquals({'f': 'v'}, self.client.get(index='i', doc_type='t', id=47)['_source']) + self.assertFalse(self.client.exists(index="i", id=45)) + self.assertEquals({"answer": 42}, self.client.get(index="i", id=42)["_source"]) + self.assertEquals({"f": "v"}, self.client.get(index="i", id=47)["_source"]) def test_transport_error_can_becaught(self): failing_client = FailingBulkClient(self.client) docs = [ - {'_index': 'i', '_type': 't', '_id': 47, 'f': 'v'}, - {'_index': 'i', '_type': 't', '_id': 45, 'f': 'v'}, - {'_index': 'i', '_type': 't', '_id': 42, 'f': 'v'}, + {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"}, + {"_index": "i", "_type": "_doc", "_id": 45, "f": "v"}, + {"_index": "i", "_type": "_doc", "_id": 42, "f": "v"}, ] - results = list(helpers.streaming_bulk(failing_client, docs, raise_on_exception=False, raise_on_error=False, chunk_size=1)) + results = list( + helpers.streaming_bulk( + failing_client, + docs, + raise_on_exception=False, + raise_on_error=False, + chunk_size=1, + ) + ) self.assertEquals(3, len(results)) self.assertEquals([True, False, True], [r[0] for r in results]) - exc = results[1][1]['index'].pop('exception') + exc = results[1][1]["index"].pop("exception") self.assertIsInstance(exc, TransportError) self.assertEquals(599, exc.status_code) self.assertEquals( { - 'index': { - '_index': 'i', - '_type': 't', - '_id': 45, - 'data': {'f': 'v'}, - 'error': "TransportError(599, 'Error!')", - 'status': 599 + "index": { + "_index": "i", + "_type": "_doc", + "_id": 45, + "data": {"f": "v"}, + "error": "TransportError(599, 'Error!')", + "status": 599, } }, - results[1][1] + results[1][1], ) def test_rejected_documents_are_retried(self): - failing_client = FailingBulkClient(self.client, fail_with=TransportError(429, 'Rejected!', {})) + failing_client = FailingBulkClient( + self.client, fail_with=TransportError(429, "Rejected!", {}) + ) docs = [ - {'_index': 'i', '_type': 't', '_id': 47, 'f': 'v'}, - {'_index': 'i', '_type': 't', '_id': 45, 'f': 'v'}, - {'_index': 'i', '_type': 't', '_id': 42, 'f': 'v'}, + {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"}, + {"_index": "i", "_type": "_doc", "_id": 45, "f": "v"}, + {"_index": "i", "_type": "_doc", "_id": 42, "f": "v"}, ] - results = list(helpers.streaming_bulk(failing_client, docs, - raise_on_exception=False, - raise_on_error=False, - chunk_size=1, max_retries=1, - initial_backoff=0)) + results = list( + helpers.streaming_bulk( + failing_client, + docs, + raise_on_exception=False, + raise_on_error=False, + chunk_size=1, + max_retries=1, + initial_backoff=0, + ) + ) self.assertEquals(3, len(results)) self.assertEquals([True, True, True], [r[0] for r in results]) - self.client.indices.refresh(index='i') - res = self.client.search(index='i') - self.assertEquals(3, res['hits']['total']) + self.client.indices.refresh(index="i") + res = self.client.search(index="i") + self.assertEquals({"value": 3, "relation": "eq"}, res["hits"]["total"]) self.assertEquals(4, failing_client._called) def test_rejected_documents_are_retried_at_most_max_retries_times(self): - failing_client = FailingBulkClient(self.client, fail_at=(1, 2, ), - fail_with=TransportError(429, 'Rejected!', {})) + failing_client = FailingBulkClient( + self.client, fail_at=(1, 2), fail_with=TransportError(429, "Rejected!", {}) + ) docs = [ - {'_index': 'i', '_type': 't', '_id': 47, 'f': 'v'}, - {'_index': 'i', '_type': 't', '_id': 45, 'f': 'v'}, - {'_index': 'i', '_type': 't', '_id': 42, 'f': 'v'}, + {"_index": "i", "_type": "_doc", "_id": 47, "f": "v"}, + {"_index": "i", "_type": "_doc", "_id": 45, "f": "v"}, + {"_index": "i", "_type": "_doc", "_id": 42, "f": "v"}, ] - results = list(helpers.streaming_bulk(failing_client, docs, - raise_on_exception=False, - raise_on_error=False, - chunk_size=1, max_retries=1, - initial_backoff=0)) + results = list( + helpers.streaming_bulk( + failing_client, + docs, + raise_on_exception=False, + raise_on_error=False, + chunk_size=1, + max_retries=1, + initial_backoff=0, + ) + ) self.assertEquals(3, len(results)) self.assertEquals([False, True, True], [r[0] for r in results]) - self.client.indices.refresh(index='i') - res = self.client.search(index='i') - self.assertEquals(2, res['hits']['total']) + self.client.indices.refresh(index="i") + res = self.client.search(index="i") + self.assertEquals({"value": 2, "relation": "eq"}, res["hits"]["total"]) self.assertEquals(4, failing_client._called) def test_transport_error_is_raised_with_max_retries(self): - failing_client = FailingBulkClient(self.client, fail_at=(1, 2, 3, 4, ), - fail_with=TransportError(429, 'Rejected!', {})) + failing_client = FailingBulkClient( + self.client, + fail_at=(1, 2, 3, 4), + fail_with=TransportError(429, "Rejected!", {}), + ) def streaming_bulk(): - results = list(helpers.streaming_bulk( - failing_client, - [{"a": 42}, {"a": 39}], - raise_on_exception=True, - max_retries=3, - initial_backoff=0 - )) + results = list( + helpers.streaming_bulk( + failing_client, + [{"a": 42}, {"a": 39}], + raise_on_exception=True, + max_retries=3, + initial_backoff=0, + ) + ) return results self.assertRaises(TransportError, streaming_bulk) @@ -156,85 +202,103 @@ class TestStreamingBulk(ElasticsearchTestCase): class TestBulk(ElasticsearchTestCase): def test_bulk_works_with_single_item(self): - docs = [{"answer": 42, '_id': 1}] - success, failed = helpers.bulk(self.client, docs, index='test-index', doc_type='answers', refresh=True) + docs = [{"answer": 42, "_id": 1}] + success, failed = helpers.bulk( + self.client, docs, index="test-index", refresh=True + ) self.assertEquals(1, success) self.assertFalse(failed) - self.assertEquals(1, self.client.count(index='test-index', doc_type='answers')['count']) - self.assertEquals({"answer": 42}, self.client.get(index='test-index', doc_type='answers', id=1)['_source']) + self.assertEquals(1, self.client.count(index="test-index")["count"]) + self.assertEquals( + {"answer": 42}, self.client.get(index="test-index", id=1)["_source"] + ) def test_all_documents_get_inserted(self): - docs = [{"answer": x, '_id': x} for x in range(100)] - success, failed = helpers.bulk(self.client, docs, index='test-index', doc_type='answers', refresh=True) + docs = [{"answer": x, "_id": x} for x in range(100)] + success, failed = helpers.bulk( + self.client, docs, index="test-index", refresh=True + ) self.assertEquals(100, success) self.assertFalse(failed) - self.assertEquals(100, self.client.count(index='test-index', doc_type='answers')['count']) - self.assertEquals({"answer": 42}, self.client.get(index='test-index', doc_type='answers', id=42)['_source']) + self.assertEquals(100, self.client.count(index="test-index")["count"]) + self.assertEquals( + {"answer": 42}, self.client.get(index="test-index", id=42)["_source"] + ) def test_stats_only_reports_numbers(self): docs = [{"answer": x} for x in range(100)] - success, failed = helpers.bulk(self.client, docs, index='test-index', doc_type='answers', refresh=True, stats_only=True) + success, failed = helpers.bulk( + self.client, docs, index="test-index", refresh=True, stats_only=True + ) self.assertEquals(100, success) self.assertEquals(0, failed) - self.assertEquals(100, self.client.count(index='test-index', doc_type='answers')['count']) + self.assertEquals(100, self.client.count(index="test-index")["count"]) def test_errors_are_reported_correctly(self): - self.client.indices.create("i", + self.client.indices.create( + "i", { - "mappings": {"t": {"properties": {"a": {"type": "integer"}}}}, - "settings": {"number_of_shards": 1, "number_of_replicas": 0} - }) + "mappings": {"properties": {"a": {"type": "integer"}}}, + "settings": {"number_of_shards": 1, "number_of_replicas": 0}, + }, + ) self.client.cluster.health(wait_for_status="yellow") success, failed = helpers.bulk( self.client, - [{"a": 42}, {"a": "c", '_id': 42}], + [{"a": 42}, {"a": "c", "_id": 42}], index="i", - doc_type="t", - raise_on_error=False + raise_on_error=False, ) self.assertEquals(1, success) self.assertEquals(1, len(failed)) error = failed[0] - self.assertEquals('42', error['index']['_id']) - self.assertEquals('t', error['index']['_type']) - self.assertEquals('i', error['index']['_index']) - print(error['index']['error']) - self.assertTrue('MapperParsingException' in repr(error['index']['error']) or 'mapper_parsing_exception' in repr(error['index']['error'])) + self.assertEquals("42", error["index"]["_id"]) + self.assertEquals("_doc", error["index"]["_type"]) + self.assertEquals("i", error["index"]["_index"]) + print(error["index"]["error"]) + self.assertTrue( + "MapperParsingException" in repr(error["index"]["error"]) + or "mapper_parsing_exception" in repr(error["index"]["error"]) + ) def test_error_is_raised(self): - self.client.indices.create("i", + self.client.indices.create( + "i", { - "mappings": {"t": {"properties": {"a": {"type": "integer"}}}}, - "settings": {"number_of_shards": 1, "number_of_replicas": 0} - }) + "mappings": {"properties": {"a": {"type": "integer"}}}, + "settings": {"number_of_shards": 1, "number_of_replicas": 0}, + }, + ) self.client.cluster.health(wait_for_status="yellow") - self.assertRaises(helpers.BulkIndexError, helpers.bulk, + self.assertRaises( + helpers.BulkIndexError, + helpers.bulk, self.client, [{"a": 42}, {"a": "c"}], index="i", - doc_type="t" ) def test_errors_are_collected_properly(self): - self.client.indices.create("i", + self.client.indices.create( + "i", { - "mappings": {"t": {"properties": {"a": {"type": "integer"}}}}, - "settings": {"number_of_shards": 1, "number_of_replicas": 0} - }) + "mappings": {"properties": {"a": {"type": "integer"}}}, + "settings": {"number_of_shards": 1, "number_of_replicas": 0}, + }, + ) self.client.cluster.health(wait_for_status="yellow") success, failed = helpers.bulk( self.client, [{"a": 42}, {"a": "c"}], index="i", - doc_type="t", stats_only=True, - raise_on_error=False + raise_on_error=False, ) self.assertEquals(1, success) self.assertEquals(1, failed) @@ -244,131 +308,168 @@ class TestScan(ElasticsearchTestCase): def test_order_can_be_preserved(self): bulk = [] for x in range(100): - bulk.append({"index": {"_index": "test_index", "_type": "answers", "_id": x}}) + bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}}) bulk.append({"answer": x, "correct": x == 42}) self.client.bulk(bulk, refresh=True) - docs = list(helpers.scan(self.client, index="test_index", doc_type="answers", query={"sort": "answer"}, preserve_order=True)) + docs = list( + helpers.scan( + self.client, + index="test_index", + query={"sort": "answer"}, + preserve_order=True, + ) + ) self.assertEquals(100, len(docs)) - self.assertEquals(list(map(str, range(100))), list(d['_id'] for d in docs)) - self.assertEquals(list(range(100)), list(d['_source']['answer'] for d in docs)) + self.assertEquals(list(map(str, range(100))), list(d["_id"] for d in docs)) + self.assertEquals(list(range(100)), list(d["_source"]["answer"] for d in docs)) def test_all_documents_are_read(self): bulk = [] for x in range(100): - bulk.append({"index": {"_index": "test_index", "_type": "answers", "_id": x}}) + bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}}) bulk.append({"answer": x, "correct": x == 42}) self.client.bulk(bulk, refresh=True) - docs = list(helpers.scan(self.client, index="test_index", doc_type="answers", size=2)) + docs = list(helpers.scan(self.client, index="test_index", size=2)) self.assertEquals(100, len(docs)) - self.assertEquals(set(map(str, range(100))), set(d['_id'] for d in docs)) - self.assertEquals(set(range(100)), set(d['_source']['answer'] for d in docs)) + self.assertEquals(set(map(str, range(100))), set(d["_id"] for d in docs)) + self.assertEquals(set(range(100)), set(d["_source"]["answer"] for d in docs)) + class TestReindex(ElasticsearchTestCase): def setUp(self): super(TestReindex, self).setUp() bulk = [] for x in range(100): - bulk.append({"index": {"_index": "test_index", "_type": "post", "_id": x}}) - bulk.append({"answer": x, "correct": x == 42, "type": "answers" if x % 2 == 0 else "questions"}) + bulk.append({"index": {"_index": "test_index", "_type": "_doc", "_id": x}}) + bulk.append( + { + "answer": x, + "correct": x == 42, + "type": "answers" if x % 2 == 0 else "questions", + } + ) self.client.bulk(bulk, refresh=True) def test_reindex_passes_kwargs_to_scan_and_bulk(self): - helpers.reindex(self.client, "test_index", "prod_index", scan_kwargs={'q': 'type:answers'}, bulk_kwargs={'refresh': True}) + helpers.reindex( + self.client, + "test_index", + "prod_index", + scan_kwargs={"q": "type:answers"}, + bulk_kwargs={"refresh": True}, + ) self.assertTrue(self.client.indices.exists("prod_index")) - self.assertEquals(50, self.client.count(index='prod_index', q='type:answers')['count']) + self.assertEquals( + 50, self.client.count(index="prod_index", q="type:answers")["count"] + ) - self.assertEquals({"answer": 42, "correct": True, "type": "answers"}, self.client.get(index="prod_index", doc_type="post", id=42)['_source']) + self.assertEquals( + {"answer": 42, "correct": True, "type": "answers"}, + self.client.get(index="prod_index", id=42)["_source"], + ) def test_reindex_accepts_a_query(self): - helpers.reindex(self.client, "test_index", "prod_index", query={"query": {"bool": {"filter": {"term": {"type": "answers"}}}}}) + helpers.reindex( + self.client, + "test_index", + "prod_index", + query={"query": {"bool": {"filter": {"term": {"type": "answers"}}}}}, + ) self.client.indices.refresh() self.assertTrue(self.client.indices.exists("prod_index")) - self.assertEquals(50, self.client.count(index='prod_index', q='type:answers')['count']) + self.assertEquals( + 50, self.client.count(index="prod_index", q="type:answers")["count"] + ) - self.assertEquals({"answer": 42, "correct": True, "type": "answers"}, self.client.get(index="prod_index", doc_type="post", id=42)['_source']) + self.assertEquals( + {"answer": 42, "correct": True, "type": "answers"}, + self.client.get(index="prod_index", id=42)["_source"], + ) def test_all_documents_get_moved(self): helpers.reindex(self.client, "test_index", "prod_index") self.client.indices.refresh() self.assertTrue(self.client.indices.exists("prod_index")) - self.assertEquals(50, self.client.count(index='prod_index', q='type:questions')['count']) - self.assertEquals(50, self.client.count(index='prod_index', q='type:answers')['count']) + self.assertEquals( + 50, self.client.count(index="prod_index", q="type:questions")["count"] + ) + self.assertEquals( + 50, self.client.count(index="prod_index", q="type:answers")["count"] + ) + + self.assertEquals( + {"answer": 42, "correct": True, "type": "answers"}, + self.client.get(index="prod_index", id=42)["_source"], + ) - self.assertEquals({"answer": 42, "correct": True, "type": "answers"}, self.client.get(index="prod_index", doc_type="post", id=42)['_source']) class TestParentChildReindex(ElasticsearchTestCase): def setUp(self): super(TestParentChildReindex, self).setUp() - body={ - 'settings': {"number_of_shards": 1, "number_of_replicas": 0}, - 'mappings': { - 'post': { - 'properties': { - 'question_answer': { - 'type': 'join', - 'relations': {'question': 'answer'} - } + body = { + "settings": {"number_of_shards": 1, "number_of_replicas": 0}, + "mappings": { + "properties": { + "question_answer": { + "type": "join", + "relations": {"question": "answer"}, } } - } + }, } - self.client.indices.create(index='test-index', body=body) - self.client.indices.create(index='real-index', body=body) + self.client.indices.create(index="test-index", body=body) + self.client.indices.create(index="real-index", body=body) self.client.index( - index='test-index', - doc_type='post', - id=42, - body={'question_answer': 'question'}, + index="test-index", id=42, body={"question_answer": "question"} ) self.client.index( - index='test-index', - doc_type='post', + index="test-index", id=47, routing=42, - body={'some': 'data', 'question_answer': {'name': 'answer', 'parent': 42}}, + body={"some": "data", "question_answer": {"name": "answer", "parent": 42}}, ) - self.client.indices.refresh(index='test-index') + self.client.indices.refresh(index="test-index") def test_children_are_reindexed_correctly(self): - helpers.reindex(self.client, 'test-index', 'real-index') + helpers.reindex(self.client, "test-index", "real-index") - q = self.client.get( - index='real-index', - doc_type='post', - id=42 - ) + q = self.client.get(index="real-index", id=42) self.assertEquals( { - '_id': '42', - '_index': 'real-index', - '_source': {'question_answer': 'question'}, - '_type': 'post', - '_version': 1, - 'found': True - }, q - ) - q = self.client.get( - index='test-index', - doc_type='post', - id=47, - routing=42 + "_id": "42", + "_index": "real-index", + "_primary_term": 1, + "_seq_no": 0, + "_source": {"question_answer": "question"}, + "_type": "_doc", + "_version": 1, + "found": True, + }, + q, ) + q = self.client.get(index="test-index", id=47, routing=42) self.assertEquals( { - '_routing': '42', - '_id': '47', - '_index': 'test-index', - '_source': {'some': 'data', 'question_answer': {'name': 'answer', 'parent': 42}}, - '_type': 'post', - '_version': 1, - 'found': True - }, q + "_routing": "42", + "_id": "47", + "_index": "test-index", + "_primary_term": 1, + "_seq_no": 1, + "_source": { + "some": "data", + "question_answer": {"name": "answer", "parent": 42}, + }, + "_type": "_doc", + "_version": 1, + "found": True, + }, + q, )