From 1a74b015e8a30f18cb5acfc9a9b86724288cca01 Mon Sep 17 00:00:00 2001 From: Honza Kral Date: Fri, 14 Jun 2013 17:18:50 +0200 Subject: [PATCH] Analyze API and related test suite using the external yaml definition In the future the yaml files should be in separate repo. --- elasticsearch/client.py | 49 ++++++- .../test_server/10_analyze.yaml | 52 +++++++ test_elasticsearch/test_server/test_common.py | 129 ++++++++++++++++++ 3 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 test_elasticsearch/test_server/10_analyze.yaml create mode 100644 test_elasticsearch/test_server/test_common.py diff --git a/elasticsearch/client.py b/elasticsearch/client.py index 1bc3d7a0..a5ee16db 100644 --- a/elasticsearch/client.py +++ b/elasticsearch/client.py @@ -85,9 +85,56 @@ class NamespacedClient(object): return self.client.transport class ClusterClient(NamespacedClient): - pass + @query_params('level', 'local', 'master_timeout', 'timeout', 'wait_for_active_shards', 'wait_for_nodes', 'wait_for_relocating_shards', 'wait_for_status') + def health(self, index=None, params=None): + """ + The cluster health API allows to get a very simple status on the health of the cluster. + http://elasticsearch.org/guide/reference/api/admin-cluster-health/ + + :arg index: Limit the information returned to a specific index + :arg level: Specify the level of detail for returned information, default u'cluster' + :arg local: Return local information, do not retrieve the state from master node (default: false) + :arg master_timeout: Explicit operation timeout for connection to master node + :arg timeout: Explicit operation timeout + :arg wait_for_active_shards: Wait until the specified number of shards is active + :arg wait_for_nodes: Wait until the specified number of nodes is available + :arg wait_for_relocating_shards: Wait until the specified number of relocating shards is finished + :arg wait_for_status: Wait until cluster is in a specific state, default None + """ + status, data = self.transport.perform_request('GET', _make_path('_cluster', 'health', index), params=params) + return data class InidicesClient(NamespacedClient): + @query_params('analyzer', 'field', 'filters', 'format', 'index', 'prefer_local', 'text', 'tokenizer') + def analyze(self, index=None, body=None, params=None): + """ + Performs the analysis process on a text and return the tokens breakdown of the text. + http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze/ + + :arg index: The name of the index to scope the operation + :arg analyzer: The name of the analyzer to use + :arg field: The name of the field to + :arg filters: A comma-separated list of filters to use for the analysis + :arg index: The name of the index to scope the operation + :arg prefer_local: With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true) + :arg text: The text on which the analysis should be performed (when request body is not used) + :arg tokenizer: The name of the tokenizer to use for the analysis + """ + status, data = self.transport.perform_request('GET', _make_path(index, '_analyze'), params=params, body=body) + return data + + @query_params('ignore_indices') + def refresh(self, index=None, params=None): + """ + The refresh API allows to explicitly refresh one or more index, making all operations performed since the last refresh available for search. + http://www.elasticsearch.org/guide/reference/api/admin-indices-refresh/ + + :arg index: A comma-separated list of index names; use `_all` or empty string to perform the operation on all indices + :arg ignore_indices: When performed on multiple indices, allows to ignore `missing` ones, default u'none' + """ + status, data = self.transport.perform_request('POST', _make_path(index, '_refresh'), params=params) + return data + @query_params('timeout') def create(self, index, body=None, params=None): """ diff --git a/test_elasticsearch/test_server/10_analyze.yaml b/test_elasticsearch/test_server/10_analyze.yaml new file mode 100644 index 00000000..3d5ca473 --- /dev/null +++ b/test_elasticsearch/test_server/10_analyze.yaml @@ -0,0 +1,52 @@ +--- +"Analyze API setup": + - do: + indices.create: + index: test + body: + mappings: + test: + properties: + text: + type: string + analyzer: whitespace + +--- +"Analyze API text format": + - do: + indices.analyze: + format: text + text: tHE BLACK and white! AND red + - is: + tokens: "[black:4->9:]\n\n4: \n[white:14->19:]\n\n6: \n[red:25->28:]\n" + +--- +"Analyze API JSON format": + - do: + indices.analyze: + text: Foo Bar + - length: { tokens: 2 } + - is: { tokens.0.token: foo } + - is: { tokens.1.token: bar } + +--- +"Analyze API JSON format - tokenizer and filter": + - do: + indices.analyze: + filters: lowercase + text: Foo Bar + tokenizer: keyword + - length: { tokens: 1 } + - is: { tokens.0.token: foo bar } + +--- +"Analyze API JSON format - index and field": + - do: + indices.analyze: + field: text + index: test + text: Foo Bar! + - length: { tokens: 2 } + - is: { tokens.0.token: Foo } + - is: { tokens.1.token: Bar! } + diff --git a/test_elasticsearch/test_server/test_common.py b/test_elasticsearch/test_server/test_common.py new file mode 100644 index 00000000..277fe8aa --- /dev/null +++ b/test_elasticsearch/test_server/test_common.py @@ -0,0 +1,129 @@ +""" +Dynamically generated set of TestCases based on set of yaml files decribing +some integration tests. These files are shared among all official Elasticsearch +clients. +""" +from os import listdir +from os.path import dirname, abspath, join +import yaml +from unittest import TestCase, SkipTest + +from elasticsearch import Elasticsearch + + +class InvalidActionType(SkipTest): + pass + + +class YamlTestCase(TestCase): + def run_code(self, test): + """ Execute an instruction based on it's type. """ + for action in test: + self.assertEquals(1, len(action)) + action_type, action = action.items()[0] + + if hasattr(self, 'run_' + action_type): + getattr(self, 'run_' + action_type)(action) + else: + raise InvalidActionType(action_type) + + def run_do(self, action): + """ Perform an api call with given parameters. """ + self.assertEquals(1, len(action)) + + method, args = action.items()[0] + + # locate api endpoint + api = self.client + for m in method.split('.'): + self.assertTrue(hasattr(api, m)) + api = getattr(api, m) + + self.last_response = api(**args) + + def run_length(self, action): + self.run_is(action, len) + + def run_is(self, action, transform=None): + """ Match part of last response to test data. """ + self.assertEquals(1, len(action)) + path, expected = action.items()[0] + + # fetch the possibly nested value from last_response + value = self.last_response + for step in path.split('.'): + if step.isdigit(): + step = int(step) + self.assertIsInstance(value, list) + self.assertGreater(len(value), step) + else: + self.assertIn(step, value) + value = value[step] + + # sometimes we need to transform the json value before comparing + if transform: + value = transform(value) + + # compare target value + self.assertEquals(expected, value) + + def tearDown(self): + # clean up everything + self.client.indices.delete() + + + +def construct_case(filename, name): + """ + Parse a definition of a test case from a yaml file and construct the + TestCase subclass dynamically transforming the individual tests into test + methods. Always use the first one as `setUp`. + """ + def get_test_method(name, test): + def test_(self): + self.run_code(test) + + # remember the name as docstring so it will show up + test_.__doc__ = name + return test_ + + + def get_setUp(name, definition): + def setUp(self): + self.client = Elasticsearch(['localhost:9900']) + self.last_response = None + self.run_code(definition) + + # make sure the cluster is ready + self.client.cluster.health(wait_for_status='yellow') + self.client.indices.refresh() + + setUp.__doc__ = name + return setUp + + with open(filename) as f: + tests = list(yaml.load_all(f)) + + + # take the first test as setUp method + attrs = {'setUp' : get_setUp(*tests.pop(0).items()[0])} + # create test methods for the rest + for i, test in enumerate(tests): + if not test: + continue + attrs['test_%d' % i] = get_test_method(*test.items()[0]) + + return type(name, (YamlTestCase, ), attrs) + + +current_dir = abspath(dirname(__file__)) +# find all the test definitions in yaml files ... +for filename in listdir(current_dir): + if not filename.endswith('.yaml'): + continue + + # ... parse them + name = 'Test' + filename.rsplit('.', 1)[0][3:].title() + # and insert them into locals for test runner to find them + locals()[name] = construct_case(join(current_dir, filename), name) +