Black (#948)
This commit is contained in:
Nick Lang
+89
-86
@@ -14,65 +14,62 @@ from elasticsearch import Elasticsearch
|
||||
from elasticsearch.exceptions import TransportError
|
||||
from elasticsearch.helpers import bulk, streaming_bulk
|
||||
|
||||
|
||||
def create_git_index(client, index):
|
||||
# we will use user on several places
|
||||
user_mapping = {
|
||||
'properties': {
|
||||
'name': {
|
||||
'type': 'text',
|
||||
'fields': {
|
||||
'keyword': {'type': 'keyword'},
|
||||
}
|
||||
"properties": {
|
||||
"name": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
create_index_body = {
|
||||
'settings': {
|
||||
# just one shard, no replicas for testing
|
||||
'number_of_shards': 1,
|
||||
'number_of_replicas': 0,
|
||||
|
||||
# custom analyzer for analyzing file paths
|
||||
'analysis': {
|
||||
'analyzer': {
|
||||
'file_path': {
|
||||
'type': 'custom',
|
||||
'tokenizer': 'path_hierarchy',
|
||||
'filter': ['lowercase']
|
||||
"settings": {
|
||||
# just one shard, no replicas for testing
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
# custom analyzer for analyzing file paths
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"file_path": {
|
||||
"type": "custom",
|
||||
"tokenizer": "path_hierarchy",
|
||||
"filter": ["lowercase"],
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"mappings": {
|
||||
"doc": {
|
||||
"properties": {
|
||||
"repository": {"type": "keyword"},
|
||||
"author": user_mapping,
|
||||
"authored_date": {"type": "date"},
|
||||
"committer": user_mapping,
|
||||
"committed_date": {"type": "date"},
|
||||
"parent_shas": {"type": "keyword"},
|
||||
"description": {"type": "text", "analyzer": "snowball"},
|
||||
"files": {
|
||||
"type": "text",
|
||||
"analyzer": "file_path",
|
||||
"fielddata": True,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
'mappings': {
|
||||
'doc': {
|
||||
'properties': {
|
||||
'repository': {'type': 'keyword'},
|
||||
'author': user_mapping,
|
||||
'authored_date': {'type': 'date'},
|
||||
'committer': user_mapping,
|
||||
'committed_date': {'type': 'date'},
|
||||
'parent_shas': {'type': 'keyword'},
|
||||
'description': {'type': 'text', 'analyzer': 'snowball'},
|
||||
'files': {'type': 'text', 'analyzer': 'file_path', "fielddata": True}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
# create empty index
|
||||
try:
|
||||
client.indices.create(
|
||||
index=index,
|
||||
body=create_index_body,
|
||||
)
|
||||
client.indices.create(index=index, body=create_index_body)
|
||||
except TransportError as e:
|
||||
# ignore already existing index
|
||||
if e.error == 'index_already_exists_exception':
|
||||
if e.error == "index_already_exists_exception":
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def parse_commits(head, name):
|
||||
"""
|
||||
Go through the git repository log and generate a document per commit
|
||||
@@ -80,26 +77,24 @@ def parse_commits(head, name):
|
||||
"""
|
||||
for commit in head.traverse():
|
||||
yield {
|
||||
'_id': commit.hexsha,
|
||||
'repository': name,
|
||||
'committed_date': datetime.fromtimestamp(commit.committed_date),
|
||||
'committer': {
|
||||
'name': commit.committer.name,
|
||||
'email': commit.committer.email,
|
||||
"_id": commit.hexsha,
|
||||
"repository": name,
|
||||
"committed_date": datetime.fromtimestamp(commit.committed_date),
|
||||
"committer": {
|
||||
"name": commit.committer.name,
|
||||
"email": commit.committer.email,
|
||||
},
|
||||
'authored_date': datetime.fromtimestamp(commit.authored_date),
|
||||
'author': {
|
||||
'name': commit.author.name,
|
||||
'email': commit.author.email,
|
||||
},
|
||||
'description': commit.message,
|
||||
'parent_shas': [p.hexsha for p in commit.parents],
|
||||
"authored_date": datetime.fromtimestamp(commit.authored_date),
|
||||
"author": {"name": commit.author.name, "email": commit.author.email},
|
||||
"description": commit.message,
|
||||
"parent_shas": [p.hexsha for p in commit.parents],
|
||||
# we only care about the filenames, not the per-file stats
|
||||
'files': list(commit.stats.files),
|
||||
'stats': commit.stats.total,
|
||||
"files": list(commit.stats.files),
|
||||
"stats": commit.stats.total,
|
||||
}
|
||||
|
||||
def load_repo(client, path=None, index='git'):
|
||||
|
||||
def load_repo(client, path=None, index="git"):
|
||||
"""
|
||||
Parse a git repository with all it's commits and load it into elasticsearch
|
||||
using `client`. If the index doesn't exist it will be created.
|
||||
@@ -114,18 +109,18 @@ def load_repo(client, path=None, index='git'):
|
||||
# in - since the `parse_commits` function is a generator this will avoid
|
||||
# loading all the commits into memory
|
||||
for ok, result in streaming_bulk(
|
||||
client,
|
||||
parse_commits(repo.refs.master.commit, repo_name),
|
||||
index=index,
|
||||
doc_type='doc',
|
||||
chunk_size=50 # keep the batch sizes small for appearances only
|
||||
):
|
||||
client,
|
||||
parse_commits(repo.refs.master.commit, repo_name),
|
||||
index=index,
|
||||
doc_type="doc",
|
||||
chunk_size=50, # keep the batch sizes small for appearances only
|
||||
):
|
||||
action, result = result.popitem()
|
||||
doc_id = '/%s/doc/%s' % (index, result['_id'])
|
||||
doc_id = "/%s/doc/%s" % (index, result["_id"])
|
||||
# process the information from ES whether the document has been
|
||||
# successfully indexed
|
||||
if not ok:
|
||||
print('Failed to %s document %s: %r' % (action, doc_id, result))
|
||||
print("Failed to %s document %s: %r" % (action, doc_id, result))
|
||||
else:
|
||||
print(doc_id)
|
||||
|
||||
@@ -133,36 +128,40 @@ def load_repo(client, path=None, index='git'):
|
||||
# we manually update some documents to add additional information
|
||||
UPDATES = [
|
||||
{
|
||||
'_type': 'doc',
|
||||
'_id': '20fbba1230cabbc0f4644f917c6c2be52b8a63e8',
|
||||
'_op_type': 'update',
|
||||
'doc': {'initial_commit': True}
|
||||
"_type": "doc",
|
||||
"_id": "20fbba1230cabbc0f4644f917c6c2be52b8a63e8",
|
||||
"_op_type": "update",
|
||||
"doc": {"initial_commit": True},
|
||||
},
|
||||
{
|
||||
'_type': 'doc',
|
||||
'_id': 'ae0073c8ca7e24d237ffd56fba495ed409081bf4',
|
||||
'_op_type': 'update',
|
||||
'doc': {'release': '5.0.0'}
|
||||
"_type": "doc",
|
||||
"_id": "ae0073c8ca7e24d237ffd56fba495ed409081bf4",
|
||||
"_op_type": "update",
|
||||
"doc": {"release": "5.0.0"},
|
||||
},
|
||||
]
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
# get trace logger and set level
|
||||
tracer = logging.getLogger('elasticsearch.trace')
|
||||
tracer = logging.getLogger("elasticsearch.trace")
|
||||
tracer.setLevel(logging.INFO)
|
||||
tracer.addHandler(logging.FileHandler('/tmp/es_trace.log'))
|
||||
tracer.addHandler(logging.FileHandler("/tmp/es_trace.log"))
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-H", "--host",
|
||||
"-H",
|
||||
"--host",
|
||||
action="store",
|
||||
default="localhost:9200",
|
||||
help="The elasticsearch host you wish to connect to. (Default: localhost:9200)")
|
||||
help="The elasticsearch host you wish to connect to. (Default: localhost:9200)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p", "--path",
|
||||
"-p",
|
||||
"--path",
|
||||
action="store",
|
||||
default=None,
|
||||
help="Path to git repo. Commits used as data to load into Elasticsearch. (Default: None")
|
||||
help="Path to git repo. Commits used as data to load into Elasticsearch. (Default: None",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -173,15 +172,19 @@ if __name__ == '__main__':
|
||||
load_repo(es, path=args.path)
|
||||
|
||||
# run the bulk operations
|
||||
success, _ = bulk(es, UPDATES, index='git')
|
||||
print('Performed %d actions' % success)
|
||||
success, _ = bulk(es, UPDATES, index="git")
|
||||
print("Performed %d actions" % success)
|
||||
|
||||
# we can now make docs visible for searching
|
||||
es.indices.refresh(index='git')
|
||||
es.indices.refresh(index="git")
|
||||
|
||||
# now we can retrieve the documents
|
||||
initial_commit = es.get(index='git', doc_type='doc', id='20fbba1230cabbc0f4644f917c6c2be52b8a63e8')
|
||||
print('%s: %s' % (initial_commit['_id'], initial_commit['_source']['committed_date']))
|
||||
initial_commit = es.get(
|
||||
index="git", doc_type="doc", id="20fbba1230cabbc0f4644f917c6c2be52b8a63e8"
|
||||
)
|
||||
print(
|
||||
"%s: %s" % (initial_commit["_id"], initial_commit["_source"]["committed_date"])
|
||||
)
|
||||
|
||||
# and now we can count the documents
|
||||
print(es.count(index='git')['count'], 'documents in index')
|
||||
print(es.count(index="git")["count"], "documents in index")
|
||||
|
||||
+66
-69
@@ -6,95 +6,92 @@ from dateutil.parser import parse as parse_date
|
||||
|
||||
from elasticsearch import Elasticsearch
|
||||
|
||||
|
||||
def print_search_stats(results):
|
||||
print('=' * 80)
|
||||
print('Total %d found in %dms' % (results['hits']['total'], results['took']))
|
||||
print('-' * 80)
|
||||
print("=" * 80)
|
||||
print("Total %d found in %dms" % (results["hits"]["total"], results["took"]))
|
||||
print("-" * 80)
|
||||
|
||||
|
||||
def print_hits(results):
|
||||
" Simple utility function to print results of a search query. "
|
||||
print_search_stats(results)
|
||||
for hit in results['hits']['hits']:
|
||||
for hit in results["hits"]["hits"]:
|
||||
# get created date for a repo and fallback to authored_date for a commit
|
||||
created_at = parse_date(hit['_source'].get('created_at', hit['_source']['authored_date']))
|
||||
print('/%s/%s/%s (%s): %s' % (
|
||||
hit['_index'], hit['_type'], hit['_id'],
|
||||
created_at.strftime('%Y-%m-%d'),
|
||||
hit['_source']['description'].split('\n')[0]))
|
||||
created_at = parse_date(
|
||||
hit["_source"].get("created_at", hit["_source"]["authored_date"])
|
||||
)
|
||||
print(
|
||||
"/%s/%s/%s (%s): %s"
|
||||
% (
|
||||
hit["_index"],
|
||||
hit["_type"],
|
||||
hit["_id"],
|
||||
created_at.strftime("%Y-%m-%d"),
|
||||
hit["_source"]["description"].split("\n")[0],
|
||||
)
|
||||
)
|
||||
|
||||
print('=' * 80)
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
|
||||
# get trace logger and set level
|
||||
tracer = logging.getLogger('elasticsearch.trace')
|
||||
tracer = logging.getLogger("elasticsearch.trace")
|
||||
tracer.setLevel(logging.INFO)
|
||||
tracer.addHandler(logging.FileHandler('/tmp/es_trace.log'))
|
||||
tracer.addHandler(logging.FileHandler("/tmp/es_trace.log"))
|
||||
# instantiate es client, connects to localhost:9200 by default
|
||||
es = Elasticsearch()
|
||||
|
||||
print('Empty search:')
|
||||
print_hits(es.search(index='git'))
|
||||
print("Empty search:")
|
||||
print_hits(es.search(index="git"))
|
||||
|
||||
print('Find commits that says "fix" without touching tests:')
|
||||
result = es.search(
|
||||
index='git',
|
||||
doc_type='doc',
|
||||
index="git",
|
||||
doc_type="doc",
|
||||
body={
|
||||
'query': {
|
||||
'bool': {
|
||||
'must': {
|
||||
'match': {'description': 'fix'}
|
||||
},
|
||||
'must_not': {
|
||||
'term': {'files': 'test_elasticsearch'}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
print_hits(result)
|
||||
|
||||
print('Last 8 Commits for elasticsearch-py:')
|
||||
result = es.search(
|
||||
index='git',
|
||||
doc_type='doc',
|
||||
body={
|
||||
'query': {
|
||||
'term': {
|
||||
'repository': 'elasticsearch-py'
|
||||
}
|
||||
},
|
||||
'sort': [
|
||||
{'committed_date': {'order': 'desc'}}
|
||||
],
|
||||
'size': 8
|
||||
}
|
||||
)
|
||||
print_hits(result)
|
||||
|
||||
print('Stats for top 10 committers:')
|
||||
result = es.search(
|
||||
index='git',
|
||||
doc_type='doc',
|
||||
body={
|
||||
'size': 0,
|
||||
'aggs': {
|
||||
'committers': {
|
||||
'terms': {
|
||||
'field': 'committer.name.keyword',
|
||||
},
|
||||
'aggs': {
|
||||
'line_stats': {
|
||||
'stats': {'field': 'stats.lines'}
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": {"match": {"description": "fix"}},
|
||||
"must_not": {"term": {"files": "test_elasticsearch"}},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
print_hits(result)
|
||||
|
||||
print("Last 8 Commits for elasticsearch-py:")
|
||||
result = es.search(
|
||||
index="git",
|
||||
doc_type="doc",
|
||||
body={
|
||||
"query": {"term": {"repository": "elasticsearch-py"}},
|
||||
"sort": [{"committed_date": {"order": "desc"}}],
|
||||
"size": 8,
|
||||
},
|
||||
)
|
||||
print_hits(result)
|
||||
|
||||
print("Stats for top 10 committers:")
|
||||
result = es.search(
|
||||
index="git",
|
||||
doc_type="doc",
|
||||
body={
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"committers": {
|
||||
"terms": {"field": "committer.name.keyword"},
|
||||
"aggs": {"line_stats": {"stats": {"field": "stats.lines"}}},
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
print_search_stats(result)
|
||||
for committer in result['aggregations']['committers']['buckets']:
|
||||
print('%15s: %3d commits changing %6d lines' % (
|
||||
committer['key'], committer['doc_count'], committer['line_stats']['sum']))
|
||||
print('=' * 80)
|
||||
for committer in result["aggregations"]["committers"]["buckets"]:
|
||||
print(
|
||||
"%15s: %3d commits changing %6d lines"
|
||||
% (committer["key"], committer["doc_count"], committer["line_stats"]["sum"])
|
||||
)
|
||||
print("=" * 80)
|
||||
|
||||
Reference in New Issue
Block a user