From 5f199882d8cec9cde5f6bdd3e65616596e741b6b Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Thu, 1 Dec 2016 10:51:04 -0500 Subject: [PATCH 1/7] multi_field type is removed from 5, move to new format Instead of ignoring all 400s when creating the index, only ignore index_already_exists_exception and raise other exceptions. --- example/load.py | 115 +++++++++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/example/load.py b/example/load.py index 6b6d67e6..20460ccf 100644 --- a/example/load.py +++ b/example/load.py @@ -10,6 +10,7 @@ import logging import git from elasticsearch import Elasticsearch +from elasticsearch.exceptions import TransportError from elasticsearch.helpers import bulk, streaming_bulk def create_git_index(client, index): @@ -17,69 +18,75 @@ def create_git_index(client, index): user_mapping = { 'properties': { 'name': { - 'type': 'multi_field', + 'type': 'string', 'fields': { 'raw': {'type' : 'string', 'index' : 'not_analyzed'}, - 'name': {'type' : 'string'} + } + } + } + } + + create_index_body = { + 'settings': { + # just one shard, no replicas for testing + 'number_of_shards': 1, + 'number_of_replicas': 0, + + # custom analyzer for analyzing file paths + 'analysis': { + 'analyzer': { + 'file_path': { + 'type': 'custom', + 'tokenizer': 'path_hierarchy', + 'filter': ['lowercase'] + } + } + } + }, + 'mappings': { + 'commits': { + '_parent': { + 'type': 'repos' + }, + 'properties': { + 'author': user_mapping, + 'authored_date': {'type': 'date'}, + 'committer': user_mapping, + 'committed_date': {'type': 'date'}, + 'parent_shas': {'type': 'string', 'index' : 'not_analyzed'}, + 'description': {'type': 'string', 'analyzer': 'snowball'}, + 'files': {'type': 'string', 'analyzer': 'file_path'} + } + }, + 'repos': { + 'properties': { + 'owner': user_mapping, + 'created_at': {'type': 'date'}, + 'description': { + 'type': 'string', + 'analyzer': 'snowball', + }, + 'tags': { + 'type': 'string', + 'index': 'not_analyzed' + } } } } } # create empty index - client.indices.create( - index=index, - body={ - 'settings': { - # just one shard, no replicas for testing - 'number_of_shards': 1, - 'number_of_replicas': 0, - - # custom analyzer for analyzing file paths - 'analysis': { - 'analyzer': { - 'file_path': { - 'type': 'custom', - 'tokenizer': 'path_hierarchy', - 'filter': ['lowercase'] - } - } - } - }, - 'mappings': { - 'commits': { - '_parent': { - 'type': 'repos' - }, - 'properties': { - 'author': user_mapping, - 'authored_date': {'type': 'date'}, - 'committer': user_mapping, - 'committed_date': {'type': 'date'}, - 'parent_shas': {'type': 'string', 'index' : 'not_analyzed'}, - 'description': {'type': 'string', 'analyzer': 'snowball'}, - 'files': {'type': 'string', 'analyzer': 'file_path'} - } - }, - 'repos': { - 'properties': { - 'owner': user_mapping, - 'created_at': {'type': 'date'}, - 'description': { - 'type': 'string', - 'analyzer': 'snowball', - }, - 'tags': { - 'type': 'string', - 'index': 'not_analyzed' - } - } - } - } - }, + try: + client.indices.create( + index=index, + body=create_index_body, + ) + except TransportError, e: # ignore already existing index - ignore=400 - ) + if e.error == 'index_already_exists_exception': + pass + else: + raise def parse_commits(head, name): """ From 9ac70f3dcb461b28c06f4cdde869351a86388718 Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Thu, 1 Dec 2016 12:08:39 -0500 Subject: [PATCH 2/7] Fix UnicodeDecodeError --- example/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/load.py b/example/load.py index 20460ccf..2cd73166 100644 --- a/example/load.py +++ b/example/load.py @@ -164,7 +164,7 @@ REPO_ACTIONS = [ }, {'_type': 'repos', '_id': 'elasticsearch-py', '_op_type': 'update', 'doc': { - 'owner': {'name': 'Honza Král', 'email': 'honza.kral@gmail.com'}, + 'owner': {'name': u'Honza Král', 'email': 'honza.kral@gmail.com'}, 'created_at': datetime(2013, 5, 1, 16, 37, 32), 'tags': ['elasticsearch', 'search', 'python', 'client'], 'description': 'For searching snakes.'} From cdd1fbf6a06ab4fe62ba0ece6c2700ca0ad6e4f9 Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Thu, 1 Dec 2016 13:45:21 -0500 Subject: [PATCH 3/7] Scripted updates are disabled by default in ES v5 --- example/load.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/example/load.py b/example/load.py index 2cd73166..d5b000c3 100644 --- a/example/load.py +++ b/example/load.py @@ -159,7 +159,7 @@ REPO_ACTIONS = [ {'_type': 'repos', '_id': 'elasticsearch', '_source': { 'owner': {'name': 'Shay Bannon', 'email': 'kimchy@gmail.com'}, 'created_at': datetime(2010, 2, 8, 15, 22, 27), - 'tags': ['search', 'distributed', 'lucene'], + 'tags': ['search', 'distributed', 'lucene', 'java'], 'description': 'You know, for search.'} }, @@ -191,19 +191,6 @@ if __name__ == '__main__': es_repo = es.get(index='git', doc_type='repos', id='elasticsearch') print('%s: %s' % (es_repo['_id'], es_repo['_source']['description'])) - # update - add java to es tags - es.update( - index='git', - doc_type='repos', - id='elasticsearch', - body={ - "script" : "ctx._source.tags += tag", - "params" : { - "tag" : "java" - } - } - ) - # refresh to make the documents available for search es.indices.refresh(index='git') From 3401b21f1cd514456574f8789485345e2bf8b2a7 Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Thu, 1 Dec 2016 13:47:08 -0500 Subject: [PATCH 4/7] Updated example queries for ES v5 --- example/queries.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/example/queries.py b/example/queries.py index 10ffcd67..d487fcb8 100644 --- a/example/queries.py +++ b/example/queries.py @@ -41,14 +41,12 @@ result = es.search( doc_type='commits', body={ 'query': { - 'filtered': { - 'query': { + 'bool': { + 'must': { 'match': {'description': 'fix'} }, - 'filter': { - 'not': { - 'term': {'files': 'test_elasticsearch'} - } + 'must_not': { + 'term': {'files': 'test_elasticsearch'} } } } @@ -62,8 +60,8 @@ result = es.search( doc_type='commits', body={ 'query': { - 'filtered': { - 'filter': { + 'bool': { + 'must': { 'term': { # parent ref is stored as type#id '_parent': 'repos#elasticsearch-py' @@ -86,19 +84,11 @@ result = es.search( body={ 'size': 0, 'query': { - 'filtered': { - 'filter': { - 'has_parent': { - 'type': 'repos', - 'query': { - 'filtered': { - 'filter': { - 'term': { - 'tags': 'python' - } - } - } - } + 'has_parent': { + 'parent_type': 'repos', + 'query': { + 'term': { + 'tags': 'python' } } } From b46f7e74e622f90cf4d9e398304d4b2fa9b7435a Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Thu, 1 Dec 2016 14:17:44 -0500 Subject: [PATCH 5/7] Querying with parent id has changed in ES v5 --- example/queries.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/example/queries.py b/example/queries.py index d487fcb8..3c12dc9f 100644 --- a/example/queries.py +++ b/example/queries.py @@ -60,13 +60,8 @@ result = es.search( doc_type='commits', body={ 'query': { - 'bool': { - 'must': { - 'term': { - # parent ref is stored as type#id - '_parent': 'repos#elasticsearch-py' - } - } + 'parent_id': { + 'type': 'commits', 'id': 'elasticsearch-py' } }, 'sort': [ From d4748c98c3e5bcd4f23a1ec8ac65b5121ca3209b Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Thu, 1 Dec 2016 16:03:46 -0500 Subject: [PATCH 6/7] Restore fixed update example in example/load.py Earlier I'd misread the Scripting and Security page about inline scripts - turns out they are enabled by default. Earlier failures I saw were due to old groovy syntax. Things are working after changing to painless. --- example/load.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/example/load.py b/example/load.py index d5b000c3..37925b93 100644 --- a/example/load.py +++ b/example/load.py @@ -159,7 +159,7 @@ REPO_ACTIONS = [ {'_type': 'repos', '_id': 'elasticsearch', '_source': { 'owner': {'name': 'Shay Bannon', 'email': 'kimchy@gmail.com'}, 'created_at': datetime(2010, 2, 8, 15, 22, 27), - 'tags': ['search', 'distributed', 'lucene', 'java'], + 'tags': ['search', 'distributed', 'lucene'], 'description': 'You know, for search.'} }, @@ -191,6 +191,21 @@ if __name__ == '__main__': es_repo = es.get(index='git', doc_type='repos', id='elasticsearch') print('%s: %s' % (es_repo['_id'], es_repo['_source']['description'])) + # update - add java to es tags + es.update( + index='git', + doc_type='repos', + id='elasticsearch', + body={ + "script": { + "inline" : "ctx._source.tags.add(params.tag)", + "params" : { + "tag" : "java" + } + } + } + ) + # refresh to make the documents available for search es.indices.refresh(index='git') From 7fbcfc2c710488ab051c7dd6d4b34426a42b4887 Mon Sep 17 00:00:00 2001 From: Jason Veatch Date: Thu, 1 Dec 2016 16:29:05 -0500 Subject: [PATCH 7/7] Cleaned up indentation in script block --- example/load.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/load.py b/example/load.py index 37925b93..0cd94c25 100644 --- a/example/load.py +++ b/example/load.py @@ -197,8 +197,8 @@ if __name__ == '__main__': doc_type='repos', id='elasticsearch', body={ - "script": { - "inline" : "ctx._source.tags.add(params.tag)", + "script": { + "inline" : "ctx._source.tags.add(params.tag)", "params" : { "tag" : "java" }