From da1a8b92c9827e0edd0cea5a67f07ada8e9237a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Kr=C3=A1l?= Date: Wed, 4 Nov 2015 18:12:36 +0100 Subject: [PATCH] Make sure example works with Elasticsearch 2.0 Fixes #300 --- example/load.py | 114 ++++++++++++++++++++------------------------- example/queries.py | 32 +++++++------ 2 files changed, 70 insertions(+), 76 deletions(-) diff --git a/example/load.py b/example/load.py index 22adf73e..6b6d67e6 100644 --- a/example/load.py +++ b/example/load.py @@ -13,6 +13,19 @@ from elasticsearch import Elasticsearch from elasticsearch.helpers import bulk, streaming_bulk def create_git_index(client, index): + # we will use user on several places + user_mapping = { + 'properties': { + 'name': { + 'type': 'multi_field', + 'fields': { + 'raw': {'type' : 'string', 'index' : 'not_analyzed'}, + 'name': {'type' : 'string'} + } + } + } + } + # create empty index client.indices.create( index=index, @@ -32,90 +45,65 @@ def create_git_index(client, index): } } } + }, + 'mappings': { + 'commits': { + '_parent': { + 'type': 'repos' + }, + 'properties': { + 'author': user_mapping, + 'authored_date': {'type': 'date'}, + 'committer': user_mapping, + 'committed_date': {'type': 'date'}, + 'parent_shas': {'type': 'string', 'index' : 'not_analyzed'}, + 'description': {'type': 'string', 'analyzer': 'snowball'}, + 'files': {'type': 'string', 'analyzer': 'file_path'} + } + }, + 'repos': { + 'properties': { + 'owner': user_mapping, + 'created_at': {'type': 'date'}, + 'description': { + 'type': 'string', + 'analyzer': 'snowball', + }, + 'tags': { + 'type': 'string', + 'index': 'not_analyzed' + } + } + } } }, # ignore already existing index ignore=400 ) - # we will use user on several places - user_mapping = { - 'properties': { - 'name': { - 'type': 'multi_field', - 'fields': { - 'raw': {'type' : 'string', 'index' : 'not_analyzed'}, - 'name': {'type' : 'string'} - } - } - } - } - - client.indices.put_mapping( - index=index, - doc_type='repos', - body={ - 'repos': { - 'properties': { - 'owner': user_mapping, - 'created_at': {'type': 'date'}, - 'description': { - 'type': 'string', - 'analyzer': 'snowball', - }, - 'tags': { - 'type': 'string', - 'index': 'not_analyzed' - } - } - } - } - ) - - client.indices.put_mapping( - index=index, - doc_type='commits', - body={ - 'commits': { - '_parent': { - 'type': 'repos' - }, - 'properties': { - 'author': user_mapping, - 'authored_date': {'type': 'date'}, - 'committer': user_mapping, - 'committed_date': {'type': 'date'}, - 'parent_shas': {'type': 'string', 'index' : 'not_analyzed'}, - 'description': {'type': 'string', 'analyzer': 'snowball'}, - 'files': {'type': 'string', 'analyzer': 'file_path'} - } - } - } - ) - -def parse_commits(repo, name): +def parse_commits(head, name): """ Go through the git repository log and generate a document per commit containing all the metadata. """ - for commit in repo.log(): + for commit in head.traverse(): yield { - '_id': commit.id, + '_id': commit.hexsha, '_parent': name, - 'committed_date': datetime(*commit.committed_date[:6]), + 'committed_date': datetime.fromtimestamp(commit.committed_date), 'committer': { 'name': commit.committer.name, 'email': commit.committer.email, }, - 'authored_date': datetime(*commit.authored_date[:6]), + 'authored_date': datetime.fromtimestamp(commit.authored_date), 'author': { 'name': commit.author.name, 'email': commit.author.email, }, 'description': commit.message, - 'parent_shas': [p.id for p in commit.parents], + 'parent_shas': [p.hexsha for p in commit.parents], # we only care about the filenames, not the per-file stats - 'files': list(chain(commit.stats.files)), + 'files': list(commit.stats.files), 'stats': commit.stats.total, } @@ -144,7 +132,7 @@ def load_repo(client, path=None, index='git'): # loading all the commits into memory for ok, result in streaming_bulk( client, - parse_commits(repo, repo_name), + parse_commits(repo.refs.master.commit, repo_name), index=index, doc_type='commits', chunk_size=50 # keep the batch sizes small for appearances only diff --git a/example/queries.py b/example/queries.py index 43b1cb1c..10ffcd67 100644 --- a/example/queries.py +++ b/example/queries.py @@ -6,12 +6,14 @@ from dateutil.parser import parse as parse_date from elasticsearch import Elasticsearch -def print_hits(results, facet_masks={}): - " Simple utility function to print results of a search query. " +def print_search_stats(results): print('=' * 80) print('Total %d found in %dms' % (results['hits']['total'], results['took'])) - if results['hits']['hits']: - print('-' * 80) + print('-' * 80) + +def print_hits(results): + " Simple utility function to print results of a search query. " + print_search_stats(results) for hit in results['hits']['hits']: # get created date for a repo and fallback to authored_date for a commit created_at = parse_date(hit['_source'].get('created_at', hit['_source']['authored_date'])) @@ -20,10 +22,6 @@ def print_hits(results, facet_masks={}): created_at.strftime('%Y-%m-%d'), hit['_source']['description'].replace('\n', ' '))) - for facet, mask in facet_masks.items(): - print('-' * 80) - for d in results['facets'][facet]['terms']: - print(mask % d) print('=' * 80) print() @@ -105,15 +103,23 @@ result = es.search( } } }, - 'facets': { + 'aggs': { 'committers': { - 'terms_stats': { - 'key_field': 'committer.name.raw', - 'value_field': 'stats.lines' + 'terms': { + 'field': 'committer.name.raw', + }, + 'aggs': { + 'line_stats': { + 'stats': {'field': 'stats.lines'} + } } } } } ) -print_hits(result, {'committers': '%(term)15s: %(count)3d commits changing %(total)6d lines'}) +print_search_stats(result) +for committer in result['aggregations']['committers']['buckets']: + print('%15s: %3d commits changing %6d lines' % ( + committer['key'], committer['doc_count'], committer['line_stats']['sum'])) +print('=' * 80)