actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Make sure example works with Elasticsearch 2.0

Browse Source 
Fixes #300
This commit is contained in:
Honza Král
2015-11-04 18:13:31 +01:00
parent 328bdb73bc
commit da1a8b92c9
2 changed files with 70 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
example/load.py
+51 -63
View File
@@ -13,6 +13,19 @@ from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk, streaming_bulk
def create_git_index(client, index):
# we will use user on several places
user_mapping = {
'properties': {
'name': {
'type': 'multi_field',
'fields': {
'raw': {'type' : 'string', 'index' : 'not_analyzed'},
'name': {'type' : 'string'}
}
}
}
}
# create empty index
client.indices.create(
index=index,
@@ -32,90 +45,65 @@ def create_git_index(client, index):
}
}
}
},
'mappings': {
'commits': {
'_parent': {
'type': 'repos'
},
'properties': {
'author': user_mapping,
'authored_date': {'type': 'date'},
'committer': user_mapping,
'committed_date': {'type': 'date'},
'parent_shas': {'type': 'string', 'index' : 'not_analyzed'},
'description': {'type': 'string', 'analyzer': 'snowball'},
'files': {'type': 'string', 'analyzer': 'file_path'}
}
},
'repos': {
'properties': {
'owner': user_mapping,
'created_at': {'type': 'date'},
'description': {
'type': 'string',
'analyzer': 'snowball',
},
'tags': {
'type': 'string',
'index': 'not_analyzed'
}
}
}
}
},
# ignore already existing index
ignore=400
)
# we will use user on several places
user_mapping = {
'properties': {
'name': {
'type': 'multi_field',
'fields': {
'raw': {'type' : 'string', 'index' : 'not_analyzed'},
'name': {'type' : 'string'}
}
}
}
}
client.indices.put_mapping(
index=index,
doc_type='repos',
body={
'repos': {
'properties': {
'owner': user_mapping,
'created_at': {'type': 'date'},
'description': {
'type': 'string',
'analyzer': 'snowball',
},
'tags': {
'type': 'string',
'index': 'not_analyzed'
}
}
}
}
)
client.indices.put_mapping(
index=index,
doc_type='commits',
body={
'commits': {
'_parent': {
'type': 'repos'
},
'properties': {
'author': user_mapping,
'authored_date': {'type': 'date'},
'committer': user_mapping,
'committed_date': {'type': 'date'},
'parent_shas': {'type': 'string', 'index' : 'not_analyzed'},
'description': {'type': 'string', 'analyzer': 'snowball'},
'files': {'type': 'string', 'analyzer': 'file_path'}
}
}
}
)
def parse_commits(repo, name):
def parse_commits(head, name):
"""
Go through the git repository log and generate a document per commit
containing all the metadata.
"""
for commit in repo.log():
for commit in head.traverse():
yield {
'_id': commit.id,
'_id': commit.hexsha,
'_parent': name,
'committed_date': datetime(*commit.committed_date[:6]),
'committed_date': datetime.fromtimestamp(commit.committed_date),
'committer': {
'name': commit.committer.name,
'email': commit.committer.email,
},
'authored_date': datetime(*commit.authored_date[:6]),
'authored_date': datetime.fromtimestamp(commit.authored_date),
'author': {
'name': commit.author.name,
'email': commit.author.email,
},
'description': commit.message,
'parent_shas': [p.id for p in commit.parents],
'parent_shas': [p.hexsha for p in commit.parents],
# we only care about the filenames, not the per-file stats
'files': list(chain(commit.stats.files)),
'files': list(commit.stats.files),
'stats': commit.stats.total,
}
@@ -144,7 +132,7 @@ def load_repo(client, path=None, index='git'):
# loading all the commits into memory
for ok, result in streaming_bulk(
client,
parse_commits(repo, repo_name),
parse_commits(repo.refs.master.commit, repo_name),
index=index,
doc_type='commits',
chunk_size=50 # keep the batch sizes small for appearances only
example/queries.py
+19 -13
View File
@@ -6,12 +6,14 @@ from dateutil.parser import parse as parse_date
from elasticsearch import Elasticsearch
def print_hits(results, facet_masks={}):
" Simple utility function to print results of a search query. "
def print_search_stats(results):
print('=' * 80)
print('Total %d found in %dms' % (results['hits']['total'], results['took']))
if results['hits']['hits']:
print('-' * 80)
print('-' * 80)
def print_hits(results):
" Simple utility function to print results of a search query. "
print_search_stats(results)
for hit in results['hits']['hits']:
# get created date for a repo and fallback to authored_date for a commit
created_at = parse_date(hit['_source'].get('created_at', hit['_source']['authored_date']))
@@ -20,10 +22,6 @@ def print_hits(results, facet_masks={}):
created_at.strftime('%Y-%m-%d'),
hit['_source']['description'].replace('\n', ' ')))
for facet, mask in facet_masks.items():
print('-' * 80)
for d in results['facets'][facet]['terms']:
print(mask % d)
print('=' * 80)
print()
@@ -105,15 +103,23 @@ result = es.search(
}
}
},
'facets': {
'aggs': {
'committers': {
'terms_stats': {
'key_field': 'committer.name.raw',
'value_field': 'stats.lines'
'terms': {
'field': 'committer.name.raw',
},
'aggs': {
'line_stats': {
'stats': {'field': 'stats.lines'}
}
}
}
}
}
)
print_hits(result, {'committers': '%(term)15s: %(count)3d commits changing %(total)6d lines'})
print_search_stats(result)
for committer in result['aggregations']['committers']['buckets']:
print('%15s: %3d commits changing %6d lines' % (
committer['key'], committer['doc_count'], committer['line_stats']['sum']))
print('=' * 80)